@lincoln504/pi-research 1.3.11 → 1.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +139 -27
- package/dist/prompts/system-coordinator.md +1 -1
- package/dist/prompts/system-lead-evaluator.md +1 -1
- package/dist/thread-worker.mjs +29 -11
- package/docs/ARCHITECTURE.md +51 -14
- package/docs/CHANGELOG.md +66 -0
- package/docs/CONFIGURATION.md +3 -2
- package/package.json +18 -10
- package/scripts/cleanup.cjs +26 -5
- package/scripts/setup.cjs +25 -1
- package/skills/pi-research/SKILL.md +1 -1
- package/src/cli.ts +36 -1
- package/src/core/service-registry.ts +19 -3
- package/src/infrastructure/browser/config.ts +36 -5
- package/src/infrastructure/browser/task-execution-service.ts +12 -2
- package/src/infrastructure/browser/thread-worker-browser.ts +34 -18
- package/src/infrastructure/browser/thread-worker.mjs +29 -11
- package/src/infrastructure/gpu-resource-service.ts +15 -1
- package/src/knowledge/embedder-init.ts +17 -1
- package/src/knowledge/store.ts +11 -5
- package/src/orchestration/research-orchestration-service.ts +22 -1
- package/src/prompts/system-coordinator.md +1 -1
- package/src/prompts/system-lead-evaluator.md +1 -1
- package/src/sdk.ts +10 -1
- package/src/security/nvd-client.ts +55 -4
- package/src/security/osv-client.ts +13 -1
- package/src/tools/research-knowledge-search.ts +18 -1
- package/src/utils/log-utils.ts +16 -1
- package/src/web-research/scraper-utils.ts +55 -14
package/dist/cli.mjs
CHANGED
|
@@ -269,7 +269,10 @@ var init_log_utils = __esm({
|
|
|
269
269
|
`${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
|
|
270
270
|
"g"
|
|
271
271
|
);
|
|
272
|
-
JWT_PATTERN = new RegExp(
|
|
272
|
+
JWT_PATTERN = new RegExp(
|
|
273
|
+
`${B_LEFT}eyJ[A-Za-z0-9_-]{1,1024}\\.[A-Za-z0-9_-]{1,8192}\\.[A-Za-z0-9_-]{1,4096}`,
|
|
274
|
+
"g"
|
|
275
|
+
);
|
|
273
276
|
BASIC_AUTH_PATTERN = new RegExp(`${B_LEFT}Basic\\s+[A-Za-z0-9+/]{16,}={0,2}`, "gi");
|
|
274
277
|
BEARER_AUTH_PATTERN = new RegExp(`${B_LEFT}Bearer\\s+[A-Za-z0-9._~+/-]+=*`, "gi");
|
|
275
278
|
PROVIDER_TOKEN_PATTERN = new RegExp(
|
|
@@ -1158,8 +1161,16 @@ var init_service_registry = __esm({
|
|
|
1158
1161
|
*/
|
|
1159
1162
|
async registerAndReplace(name, factory, options = {}) {
|
|
1160
1163
|
const mergedOptions = { ...this.defaultOptions, ...options, allowOverwrite: true };
|
|
1161
|
-
|
|
1162
|
-
|
|
1164
|
+
const registration = this.services.get(name);
|
|
1165
|
+
if (registration) {
|
|
1166
|
+
while (registration.disposalPromise) {
|
|
1167
|
+
await registration.disposalPromise.catch(() => {
|
|
1168
|
+
});
|
|
1169
|
+
}
|
|
1170
|
+
if (registration.initializationPromise) {
|
|
1171
|
+
await registration.initializationPromise.catch(() => {
|
|
1172
|
+
});
|
|
1173
|
+
}
|
|
1163
1174
|
if (registration.instance && registration.instance.dispose) {
|
|
1164
1175
|
try {
|
|
1165
1176
|
await registration.instance.dispose();
|
|
@@ -6985,7 +6996,17 @@ async function loadModelOnCPU(model, initializationTimeoutMs, useCache) {
|
|
|
6985
6996
|
interOpNumThreads: 1
|
|
6986
6997
|
}
|
|
6987
6998
|
});
|
|
6988
|
-
|
|
6999
|
+
let timedOut = true;
|
|
7000
|
+
try {
|
|
7001
|
+
const p = await withTimeout3(pipelinePromise, initializationTimeoutMs, "CPU fallback model load timed out");
|
|
7002
|
+
timedOut = false;
|
|
7003
|
+
return p;
|
|
7004
|
+
} finally {
|
|
7005
|
+
if (timedOut) {
|
|
7006
|
+
pipelinePromise.then((late) => late.dispose?.()).catch(() => {
|
|
7007
|
+
});
|
|
7008
|
+
}
|
|
7009
|
+
}
|
|
6989
7010
|
});
|
|
6990
7011
|
return loadedPipeline;
|
|
6991
7012
|
}
|
|
@@ -9182,6 +9203,7 @@ function formatQueriesBlock(queries) {
|
|
|
9182
9203
|
}
|
|
9183
9204
|
function showKnowledgeSearchWidget(ctx) {
|
|
9184
9205
|
if (ctx.mode !== "tui" || !ctx.hasUI) return;
|
|
9206
|
+
knowledgeWidgetRefCount++;
|
|
9185
9207
|
try {
|
|
9186
9208
|
const panelFactory = createKnowledgeSearchPanel();
|
|
9187
9209
|
ctx.ui.setWidget(KNOWLEDGE_WIDGET_ID, panelFactory, { placement: "aboveEditor" });
|
|
@@ -9191,6 +9213,8 @@ function showKnowledgeSearchWidget(ctx) {
|
|
|
9191
9213
|
}
|
|
9192
9214
|
function hideKnowledgeSearchWidget(ctx) {
|
|
9193
9215
|
if (ctx.mode !== "tui" || !ctx.hasUI) return;
|
|
9216
|
+
knowledgeWidgetRefCount = Math.max(0, knowledgeWidgetRefCount - 1);
|
|
9217
|
+
if (knowledgeWidgetRefCount > 0) return;
|
|
9194
9218
|
try {
|
|
9195
9219
|
ctx.ui.setWidget(KNOWLEDGE_WIDGET_ID, void 0, { placement: "aboveEditor" });
|
|
9196
9220
|
} catch (err) {
|
|
@@ -9670,7 +9694,7 @@ function createResearchKnowledgeSearchTool(iface) {
|
|
|
9670
9694
|
}
|
|
9671
9695
|
};
|
|
9672
9696
|
}
|
|
9673
|
-
var ResearchKnowledgeSearchParams, MAX_REFERENCE_CHARS, MAX_DOCUMENTS, MAX_CANDIDATES, TRIAGE_DESCRIPTION_MAX_CHARS, TRIAGE_MAX_TOKENS, KNOWLEDGE_WIDGET_ID, RESEARCH_KNOWLEDGE_MISS_STRING, RESEARCH_KNOWLEDGE_MAYBE_STRING, KNOWLEDGE_SYNTHESIS_MAX_ATTEMPTS, KNOWLEDGE_SYNTHESIS_RETRY_BASE_MS;
|
|
9697
|
+
var ResearchKnowledgeSearchParams, MAX_REFERENCE_CHARS, MAX_DOCUMENTS, MAX_CANDIDATES, TRIAGE_DESCRIPTION_MAX_CHARS, TRIAGE_MAX_TOKENS, KNOWLEDGE_WIDGET_ID, knowledgeWidgetRefCount, RESEARCH_KNOWLEDGE_MISS_STRING, RESEARCH_KNOWLEDGE_MAYBE_STRING, KNOWLEDGE_SYNTHESIS_MAX_ATTEMPTS, KNOWLEDGE_SYNTHESIS_RETRY_BASE_MS;
|
|
9674
9698
|
var init_research_knowledge_search = __esm({
|
|
9675
9699
|
"src/tools/research-knowledge-search.ts"() {
|
|
9676
9700
|
"use strict";
|
|
@@ -9705,6 +9729,7 @@ var init_research_knowledge_search = __esm({
|
|
|
9705
9729
|
TRIAGE_DESCRIPTION_MAX_CHARS = 600;
|
|
9706
9730
|
TRIAGE_MAX_TOKENS = 2048;
|
|
9707
9731
|
KNOWLEDGE_WIDGET_ID = "pi-research-knowledge-search";
|
|
9732
|
+
knowledgeWidgetRefCount = 0;
|
|
9708
9733
|
RESEARCH_KNOWLEDGE_MISS_STRING = "No results found. Live research can get the info.";
|
|
9709
9734
|
RESEARCH_KNOWLEDGE_MAYBE_STRING = "Partial results found in knowledge store. Live research can fill gaps.";
|
|
9710
9735
|
KNOWLEDGE_SYNTHESIS_MAX_ATTEMPTS = 3;
|
|
@@ -11460,9 +11485,13 @@ import { platform, homedir as homedir2 } from "node:os";
|
|
|
11460
11485
|
function getWindowsCamoufoxDir() {
|
|
11461
11486
|
return join7(homedir2(), "AppData", "Local", "camoufox", "camoufox", "Cache");
|
|
11462
11487
|
}
|
|
11488
|
+
function getCustomCamoufoxDir() {
|
|
11489
|
+
return process.env["CAMOUFOX_INSTALL_DIR"] || process.env["PLAYWRIGHT_BROWSERS_PATH"] || void 0;
|
|
11490
|
+
}
|
|
11463
11491
|
function getBrowserCacheDir() {
|
|
11464
|
-
|
|
11465
|
-
|
|
11492
|
+
const custom = getCustomCamoufoxDir();
|
|
11493
|
+
if (custom) {
|
|
11494
|
+
return custom;
|
|
11466
11495
|
}
|
|
11467
11496
|
const osPlatform = platform();
|
|
11468
11497
|
if (osPlatform === "win32") {
|
|
@@ -11493,11 +11522,13 @@ function getBrowserProfileDir(config) {
|
|
|
11493
11522
|
}
|
|
11494
11523
|
function getBrowserEnv(config) {
|
|
11495
11524
|
const env = { ...process.env };
|
|
11496
|
-
const customPath =
|
|
11525
|
+
const customPath = getCustomCamoufoxDir();
|
|
11497
11526
|
if (customPath) {
|
|
11498
11527
|
env["PLAYWRIGHT_BROWSERS_PATH"] = customPath;
|
|
11528
|
+
env["CAMOUFOX_INSTALL_DIR"] = customPath;
|
|
11499
11529
|
} else {
|
|
11500
11530
|
delete env["PLAYWRIGHT_BROWSERS_PATH"];
|
|
11531
|
+
delete env["CAMOUFOX_INSTALL_DIR"];
|
|
11501
11532
|
}
|
|
11502
11533
|
if (!env["PI_RESEARCH_LOG_FILE"]) {
|
|
11503
11534
|
const logFilePath = getLogger().getLogFilePath();
|
|
@@ -11526,7 +11557,7 @@ function ensureBrowserCacheDir() {
|
|
|
11526
11557
|
return cacheDir;
|
|
11527
11558
|
}
|
|
11528
11559
|
function getCamoufoxBinaryPath() {
|
|
11529
|
-
const customPath =
|
|
11560
|
+
const customPath = getCustomCamoufoxDir();
|
|
11530
11561
|
if (customPath) {
|
|
11531
11562
|
return customPath;
|
|
11532
11563
|
}
|
|
@@ -14304,8 +14335,22 @@ var GPUResourceService = class {
|
|
|
14304
14335
|
name = ServiceNames.GPU_RESOURCE_SERVICE;
|
|
14305
14336
|
lifecycle = "uninitialized" /* UNINITIALIZED */;
|
|
14306
14337
|
processLifecycle = null;
|
|
14307
|
-
|
|
14308
|
-
|
|
14338
|
+
/**
|
|
14339
|
+
* Backstop for a holder that is alive but wedged. Step 3 already reclaims from
|
|
14340
|
+
* *dead* owners via PID+startTime liveness, so this only ever fires against a
|
|
14341
|
+
* process we just proved is running — which makes it a lock STEAL, and the
|
|
14342
|
+
* threshold must therefore exceed the longest legitimate hold.
|
|
14343
|
+
*
|
|
14344
|
+
* It previously did not: at 5 minutes it exactly equalled the default
|
|
14345
|
+
* EMBEDDING_MODEL_INIT_TIMEOUT_MS (300000, schema max 600000), and the
|
|
14346
|
+
* embedder holds this lock across the whole model download + pipeline load.
|
|
14347
|
+
* A first-time HuggingFace fetch running near its own permitted budget would
|
|
14348
|
+
* have the GPU lock taken out from under it, admitting exactly the concurrent
|
|
14349
|
+
* WebGPU/Dawn init this lock exists to prevent. Keep this strictly above that
|
|
14350
|
+
* schema maximum.
|
|
14351
|
+
*/
|
|
14352
|
+
gpuLockStaleThresholdMs = 1e3 * 60 * 15;
|
|
14353
|
+
// 15 min > EMBEDDING_MODEL_INIT_TIMEOUT_MS max (10 min)
|
|
14309
14354
|
constructor(options) {
|
|
14310
14355
|
if (options) {
|
|
14311
14356
|
this.processLifecycle = options.processLifecycle;
|
|
@@ -17115,19 +17160,33 @@ function isSsrfBlockError(error) {
|
|
|
17115
17160
|
return false;
|
|
17116
17161
|
}
|
|
17117
17162
|
var cachedSsrfFetcher = void 0;
|
|
17118
|
-
|
|
17119
|
-
|
|
17163
|
+
function getSsrfSafeFetcher() {
|
|
17164
|
+
cachedSsrfFetcher ??= (async () => {
|
|
17165
|
+
try {
|
|
17166
|
+
const { Agent: Agent2, fetch: undiciFetch } = await import("undici");
|
|
17167
|
+
return {
|
|
17168
|
+
fetch: undiciFetch,
|
|
17169
|
+
dispatcher: new Agent2({ connect: { lookup: ssrfSafeLookup } })
|
|
17170
|
+
};
|
|
17171
|
+
} catch (e) {
|
|
17172
|
+
logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
|
|
17173
|
+
return null;
|
|
17174
|
+
}
|
|
17175
|
+
})();
|
|
17176
|
+
return cachedSsrfFetcher;
|
|
17177
|
+
}
|
|
17178
|
+
async function disposeSsrfSafeFetcher() {
|
|
17179
|
+
const pending = cachedSsrfFetcher;
|
|
17180
|
+
cachedSsrfFetcher = void 0;
|
|
17181
|
+
if (pending === void 0) return;
|
|
17182
|
+
const cached = await pending.catch(() => null);
|
|
17183
|
+
const closable = cached?.dispatcher;
|
|
17184
|
+
if (!closable?.close) return;
|
|
17120
17185
|
try {
|
|
17121
|
-
|
|
17122
|
-
cachedSsrfFetcher = {
|
|
17123
|
-
fetch: undiciFetch,
|
|
17124
|
-
dispatcher: new Agent2({ connect: { lookup: ssrfSafeLookup } })
|
|
17125
|
-
};
|
|
17186
|
+
await closable.close();
|
|
17126
17187
|
} catch (e) {
|
|
17127
|
-
logger.
|
|
17128
|
-
cachedSsrfFetcher = null;
|
|
17188
|
+
logger.debug(`[Scrapers] Error closing SSRF dispatcher: ${e instanceof Error ? e.message : String(e)}`);
|
|
17129
17189
|
}
|
|
17130
|
-
return cachedSsrfFetcher;
|
|
17131
17190
|
}
|
|
17132
17191
|
function formatErrorWithCause(err) {
|
|
17133
17192
|
try {
|
|
@@ -17986,7 +18045,6 @@ import { Value as Value7 } from "typebox/value";
|
|
|
17986
18045
|
// src/security/nvd-client.ts
|
|
17987
18046
|
init_retry_utils();
|
|
17988
18047
|
init_circuit_breaker();
|
|
17989
|
-
init_safe_unref();
|
|
17990
18048
|
init_metrics();
|
|
17991
18049
|
init_http_body();
|
|
17992
18050
|
|
|
@@ -18049,7 +18107,6 @@ var NVDRateLimiter = class {
|
|
|
18049
18107
|
signal?.removeEventListener("abort", onAbort);
|
|
18050
18108
|
resolve9();
|
|
18051
18109
|
}, waitTime);
|
|
18052
|
-
safeUnref(timeoutId);
|
|
18053
18110
|
const onAbort = () => {
|
|
18054
18111
|
clearTimeout(timeoutId);
|
|
18055
18112
|
reject(Object.assign(new Error("NVD search aborted"), { name: "AbortError" }));
|
|
@@ -18271,6 +18328,7 @@ async function fetchPaginated(term, options, maxResults, severityVersion, admitE
|
|
|
18271
18328
|
const signal = options?.signal;
|
|
18272
18329
|
let startIndex = 0;
|
|
18273
18330
|
let totalPagesFetched = 0;
|
|
18331
|
+
let seekedToNewest = false;
|
|
18274
18332
|
while (totalPagesFetched < maxPages && allVulnerabilities.length < maxResults) {
|
|
18275
18333
|
if (signal?.aborted) break;
|
|
18276
18334
|
const url = buildURL(term, options, pageSize, startIndex, severityVersion);
|
|
@@ -18291,9 +18349,17 @@ async function fetchPaginated(term, options, maxResults, severityVersion, admitE
|
|
|
18291
18349
|
break;
|
|
18292
18350
|
}
|
|
18293
18351
|
metrics.increment("nvd_cache_hits_total", 1);
|
|
18294
|
-
allVulnerabilities.push(...admitted);
|
|
18295
18352
|
const responseData = data;
|
|
18296
|
-
|
|
18353
|
+
const reportedTotal = typeof responseData?.["totalResults"] === "number" ? responseData["totalResults"] : void 0;
|
|
18354
|
+
if (!seekedToNewest && startIndex === 0 && reportedTotal !== void 0 && reportedTotal > maxResults) {
|
|
18355
|
+
seekedToNewest = true;
|
|
18356
|
+
startIndex = reportedTotal - maxResults;
|
|
18357
|
+
allVulnerabilities.length = 0;
|
|
18358
|
+
totalPagesFetched++;
|
|
18359
|
+
continue;
|
|
18360
|
+
}
|
|
18361
|
+
allVulnerabilities.push(...admitted);
|
|
18362
|
+
if (reportedTotal !== void 0 && startIndex + pageSize >= reportedTotal) {
|
|
18297
18363
|
break;
|
|
18298
18364
|
}
|
|
18299
18365
|
startIndex += pageSize;
|
|
@@ -18301,6 +18367,14 @@ async function fetchPaginated(term, options, maxResults, severityVersion, admitE
|
|
|
18301
18367
|
}
|
|
18302
18368
|
metrics.increment("nvd_pagination_requests_total", totalPagesFetched);
|
|
18303
18369
|
metrics.observe("nvd_pagination_pages_fetched", totalPagesFetched);
|
|
18370
|
+
allVulnerabilities.sort((a, b) => {
|
|
18371
|
+
const ta = a.published ? Date.parse(a.published) : NaN;
|
|
18372
|
+
const tb = b.published ? Date.parse(b.published) : NaN;
|
|
18373
|
+
if (Number.isNaN(ta) && Number.isNaN(tb)) return 0;
|
|
18374
|
+
if (Number.isNaN(ta)) return 1;
|
|
18375
|
+
if (Number.isNaN(tb)) return -1;
|
|
18376
|
+
return tb - ta;
|
|
18377
|
+
});
|
|
18304
18378
|
return allVulnerabilities.slice(0, maxResults);
|
|
18305
18379
|
}
|
|
18306
18380
|
function shouldSupplementWithV2(options) {
|
|
@@ -19311,7 +19385,14 @@ async function searchOSV(terms, options) {
|
|
|
19311
19385
|
uniqueVulns.set(vuln.id, vuln);
|
|
19312
19386
|
}
|
|
19313
19387
|
}
|
|
19314
|
-
vulnerabilities.push(...Array.from(uniqueVulns.values()))
|
|
19388
|
+
vulnerabilities.push(...Array.from(uniqueVulns.values()).sort((a, b) => {
|
|
19389
|
+
const ta = a.published ? Date.parse(a.published) : NaN;
|
|
19390
|
+
const tb = b.published ? Date.parse(b.published) : NaN;
|
|
19391
|
+
if (Number.isNaN(ta) && Number.isNaN(tb)) return 0;
|
|
19392
|
+
if (Number.isNaN(ta)) return 1;
|
|
19393
|
+
if (Number.isNaN(tb)) return -1;
|
|
19394
|
+
return tb - ta;
|
|
19395
|
+
}));
|
|
19315
19396
|
if (failures.length > 0 && failures.length === terms.length) {
|
|
19316
19397
|
throw new Error(`All OSV lookups failed: ${failures.join("; ")}`);
|
|
19317
19398
|
}
|
|
@@ -23158,6 +23239,17 @@ var ResearchOrchestrationService = class {
|
|
|
23158
23239
|
const { ctx, query, depth = 0, observer, onUpdate, sessionId, researchId, config, excludeTools } = options;
|
|
23159
23240
|
const researchConfig = config || getConfig(ctx.cwd);
|
|
23160
23241
|
const configDisabledTools = researchConfig.DISABLED_TOOLS ? researchConfig.DISABLED_TOOLS.split(",").map((s) => s.trim()).filter(Boolean) : [];
|
|
23242
|
+
for (const [surface, names] of [
|
|
23243
|
+
["excludeTools", excludeTools ?? []],
|
|
23244
|
+
["PI_RESEARCH_DISABLED_TOOLS", configDisabledTools]
|
|
23245
|
+
]) {
|
|
23246
|
+
const unknown = names.filter((t) => !RESEARCH_TOOL_NAMES.includes(t));
|
|
23247
|
+
if (unknown.length > 0) {
|
|
23248
|
+
logger.warn(
|
|
23249
|
+
`[ResearchOrchestrationService] ${surface}: ignoring unknown tool name${unknown.length === 1 ? "" : "s"} ${unknown.map((t) => `"${t}"`).join(", ")} \u2014 these exclude nothing. Valid names: ${RESEARCH_TOOL_NAMES.join(", ")}.`
|
|
23250
|
+
);
|
|
23251
|
+
}
|
|
23252
|
+
}
|
|
23161
23253
|
const mergedExcludeTools = resolveExcludedTools(excludeTools, configDisabledTools);
|
|
23162
23254
|
const selectedModel = await this.resolveResearchModel(options);
|
|
23163
23255
|
logger.info(`[ResearchOrchestrationService] Using model: ${selectedModel.provider}/${selectedModel.id}`);
|
|
@@ -24576,6 +24668,12 @@ async function _doShutdown() {
|
|
|
24576
24668
|
logger.error("[SDK] Error clearing session state:", err);
|
|
24577
24669
|
errors.push(err instanceof Error ? err : new Error(String(err)));
|
|
24578
24670
|
}
|
|
24671
|
+
try {
|
|
24672
|
+
await disposeSsrfSafeFetcher();
|
|
24673
|
+
} catch (err) {
|
|
24674
|
+
logger.error("[SDK] Error disposing SSRF fetcher:", err);
|
|
24675
|
+
errors.push(err instanceof Error ? err : new Error(String(err)));
|
|
24676
|
+
}
|
|
24579
24677
|
if (globalContainer) {
|
|
24580
24678
|
try {
|
|
24581
24679
|
await disposeCoreServices(globalContainer);
|
|
@@ -25298,6 +25396,7 @@ async function cmdStatus(json) {
|
|
|
25298
25396
|
const det = await detectCredentials();
|
|
25299
25397
|
const paths = resolvedConfigPaths();
|
|
25300
25398
|
const cfg = getConfig(process.cwd(), "cli");
|
|
25399
|
+
const browserInstalled = isBrowserAvailable();
|
|
25301
25400
|
const summary = {
|
|
25302
25401
|
package: "@lincoln504/pi-research",
|
|
25303
25402
|
version: PKG_VERSION,
|
|
@@ -25311,6 +25410,12 @@ async function cmdStatus(json) {
|
|
|
25311
25410
|
modelFrom: det.modelFrom,
|
|
25312
25411
|
problem: det.problem ?? null
|
|
25313
25412
|
},
|
|
25413
|
+
browser: {
|
|
25414
|
+
installed: browserInstalled,
|
|
25415
|
+
// Same remedy the health check prints, so both surfaces name one command.
|
|
25416
|
+
fix: browserInstalled ? null : "npx camoufox-js fetch",
|
|
25417
|
+
note: browserInstalled ? null : "Stealth browser not installed \u2014 scraping falls back to plain fetch, which some sites block. npm 11.19+ blocks install scripts by default, which skips this package's postinstall."
|
|
25418
|
+
},
|
|
25314
25419
|
knowledgeStoreMode: cfg.KNOWLEDGE_STORE_MODE,
|
|
25315
25420
|
defaultDepth: cfg.DEFAULT_RESEARCH_DEPTH,
|
|
25316
25421
|
paths
|
|
@@ -25325,6 +25430,13 @@ async function cmdStatus(json) {
|
|
|
25325
25430
|
|
|
25326
25431
|
`);
|
|
25327
25432
|
toStdout(configBlock(det, summary.ready ? void 0 : "\u2014 fix the problem above, then re-run.") + "\n");
|
|
25433
|
+
if (!summary.browser.installed) {
|
|
25434
|
+
toStdout(`
|
|
25435
|
+
stealth browser: NOT INSTALLED \u2014 run: ${summary.browser.fix}
|
|
25436
|
+
`);
|
|
25437
|
+
toStdout(` ${summary.browser.note}
|
|
25438
|
+
`);
|
|
25439
|
+
}
|
|
25328
25440
|
toStdout(`
|
|
25329
25441
|
knowledge store mode: ${cfg.KNOWLEDGE_STORE_MODE} (default depth: ${cfg.DEFAULT_RESEARCH_DEPTH})
|
|
25330
25442
|
`);
|
|
@@ -25523,7 +25635,7 @@ async function reportError(err, what, json) {
|
|
|
25523
25635
|
const isRateLimit = !isCancellation && (lower.includes("429") || lower.includes("rate limit") || lower.includes("too many requests"));
|
|
25524
25636
|
const exitCode = isCancellation ? cancellationSignal ? exitCodeForSignal(cancellationSignal) : EXIT.CANCELLED : isCapacity ? EXIT.TEMPFAIL : isConfigError && !isRateLimit ? EXIT.CONFIG : EXIT.SOFTWARE;
|
|
25525
25637
|
if (json) {
|
|
25526
|
-
const payload = { ok: false, error: msg2, exitCode };
|
|
25638
|
+
const payload = isCancellation ? { ok: false, error: `pi-research ${what} cancelled.`, cause: msg2, exitCode } : { ok: false, error: msg2, exitCode };
|
|
25527
25639
|
if ((isCapacity || isRateLimit) && !isCancellation) payload["retryable"] = true;
|
|
25528
25640
|
if (isCancellation) payload["cancelled"] = true;
|
|
25529
25641
|
if (process.env["PI_RESEARCH_DEBUG"] === "true" && err instanceof Error && err.stack) {
|
|
@@ -36,7 +36,7 @@ Before splitting the query into researchers, pin down its scope — exactly what
|
|
|
36
36
|
|
|
37
37
|
1. **Plan ONLY Round 1**: Split the query into distinct sub-topics, all inside the scope above, each assigned to a researcher. **Do NOT plan multiple rounds in advance** — additional rounds are only delegated reactively when Round 1 findings reveal gaps.
|
|
38
38
|
2. **Assign Goals**: Each researcher gets a focused goal covering a specific angle or time period.
|
|
39
|
-
3. **Temporal Awareness**: Lead with the current month and year shown above. Generate time-sensitive queries using that exact month and year (e.g., append the current year and, where relevant, the current month; "latest", "current status"). Never anchor queries to an older year
|
|
39
|
+
3. **Temporal Awareness**: Lead with the current month and year shown above. Generate time-sensitive queries using that exact month and year (e.g., append the current year and, where relevant, the current month; "latest", "current status"). Never anchor queries to an older year than the one shown above unless the user explicitly asks about the past. Research MUST be grounded in the present month and year.
|
|
40
40
|
4. **Query Planning**: For EACH researcher, generate targeted, specific queries within the budget. Fill the budget when the topic is broad enough to warrant it, but don't pad queries for a narrow topic. Each round is planned independently — make Round 1 count.
|
|
41
41
|
5. **YouTube Discovery**: For roughly **one in {{youtube_query_every_n}}** of each researcher's queries, append the word `youtube` to the query text (e.g. `"<topic> explained youtube"`). DuckDuckGo rarely surfaces YouTube on its own, and YouTube links let researchers read video transcripts. Skip this only when the topic is clearly unsuited to video (e.g. a single numeric lookup).
|
|
42
42
|
6. **Seed Search Burst**: All queries across all researchers are combined into a single pre-search pass that seeds the global link pool.
|
|
@@ -74,7 +74,7 @@ Use unique, targeted queries for any new researchers.
|
|
|
74
74
|
- **CRITICAL — Queries are mandatory**: Every researcher MUST have at least one query. Never plan a researcher without queries. Researchers receive ONLY the search results you delegate to them.
|
|
75
75
|
- **Maximize queries**: For EACH researcher, generate the maximum number of targeted, specific queries within the budget. Do not plan fewer than needed — fill the budget. Queries should target primary sources and authoritative evidence.
|
|
76
76
|
- **YouTube discovery**: For roughly one in {{youtube_query_every_n}} queries, append the word `youtube` to the query text (e.g. `"<topic> analysis youtube"`). DuckDuckGo rarely surfaces YouTube on its own, and YouTube links let researchers read video transcripts. Skip only when the gap is clearly unsuited to video.
|
|
77
|
-
- **Temporal anchoring**: Anchor every new time-sensitive query to the current month and year shown at the top of this prompt. Use that exact year (and month where relevant); never default to an older year
|
|
77
|
+
- **Temporal anchoring**: Anchor every new time-sensitive query to the current month and year shown at the top of this prompt. Use that exact year (and month where relevant); never default to an older year than the one shown there unless the user explicitly asks about the past.
|
|
78
78
|
- **Flexible coverage**: Use up to {{max_team_size}} researchers to cover distinct angles in parallel. Scale based on research gaps — a single well-targeted researcher is often sufficient for focused gaps.
|
|
79
79
|
- **Source diversity**: Encourage researchers to find multiple authoritative sources per topic area to enable comprehensive citations in the final synthesis.
|
|
80
80
|
|
package/dist/thread-worker.mjs
CHANGED
|
@@ -99,7 +99,10 @@ var init_log_utils = __esm({
|
|
|
99
99
|
`${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
|
|
100
100
|
"g"
|
|
101
101
|
);
|
|
102
|
-
JWT_PATTERN = new RegExp(
|
|
102
|
+
JWT_PATTERN = new RegExp(
|
|
103
|
+
`${B_LEFT}eyJ[A-Za-z0-9_-]{1,1024}\\.[A-Za-z0-9_-]{1,8192}\\.[A-Za-z0-9_-]{1,4096}`,
|
|
104
|
+
"g"
|
|
105
|
+
);
|
|
103
106
|
BASIC_AUTH_PATTERN = new RegExp(`${B_LEFT}Basic\\s+[A-Za-z0-9+/]{16,}={0,2}`, "gi");
|
|
104
107
|
BEARER_AUTH_PATTERN = new RegExp(`${B_LEFT}Bearer\\s+[A-Za-z0-9._~+/-]+=*`, "gi");
|
|
105
108
|
PROVIDER_TOKEN_PATTERN = new RegExp(
|
|
@@ -1509,6 +1512,7 @@ var scraper_utils_exports = {};
|
|
|
1509
1512
|
__export(scraper_utils_exports, {
|
|
1510
1513
|
createJsMarkdownConverter: () => createJsMarkdownConverter,
|
|
1511
1514
|
createNativeMarkdownConverter: () => createNativeMarkdownConverter,
|
|
1515
|
+
disposeSsrfSafeFetcher: () => disposeSsrfSafeFetcher,
|
|
1512
1516
|
extractDomain: () => extractDomain,
|
|
1513
1517
|
formatErrorWithCause: () => formatErrorWithCause,
|
|
1514
1518
|
getRandomUserAgent: () => getRandomUserAgent,
|
|
@@ -1644,19 +1648,33 @@ function isSsrfBlockError(error) {
|
|
|
1644
1648
|
}
|
|
1645
1649
|
return false;
|
|
1646
1650
|
}
|
|
1647
|
-
|
|
1648
|
-
|
|
1651
|
+
function getSsrfSafeFetcher() {
|
|
1652
|
+
cachedSsrfFetcher ??= (async () => {
|
|
1653
|
+
try {
|
|
1654
|
+
const { Agent, fetch: undiciFetch } = await import("undici");
|
|
1655
|
+
return {
|
|
1656
|
+
fetch: undiciFetch,
|
|
1657
|
+
dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } })
|
|
1658
|
+
};
|
|
1659
|
+
} catch (e) {
|
|
1660
|
+
logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
|
|
1661
|
+
return null;
|
|
1662
|
+
}
|
|
1663
|
+
})();
|
|
1664
|
+
return cachedSsrfFetcher;
|
|
1665
|
+
}
|
|
1666
|
+
async function disposeSsrfSafeFetcher() {
|
|
1667
|
+
const pending = cachedSsrfFetcher;
|
|
1668
|
+
cachedSsrfFetcher = void 0;
|
|
1669
|
+
if (pending === void 0) return;
|
|
1670
|
+
const cached = await pending.catch(() => null);
|
|
1671
|
+
const closable = cached?.dispatcher;
|
|
1672
|
+
if (!closable?.close) return;
|
|
1649
1673
|
try {
|
|
1650
|
-
|
|
1651
|
-
cachedSsrfFetcher = {
|
|
1652
|
-
fetch: undiciFetch,
|
|
1653
|
-
dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } })
|
|
1654
|
-
};
|
|
1674
|
+
await closable.close();
|
|
1655
1675
|
} catch (e) {
|
|
1656
|
-
logger.
|
|
1657
|
-
cachedSsrfFetcher = null;
|
|
1676
|
+
logger.debug(`[Scrapers] Error closing SSRF dispatcher: ${e instanceof Error ? e.message : String(e)}`);
|
|
1658
1677
|
}
|
|
1659
|
-
return cachedSsrfFetcher;
|
|
1660
1678
|
}
|
|
1661
1679
|
function formatErrorWithCause(err) {
|
|
1662
1680
|
try {
|
package/docs/ARCHITECTURE.md
CHANGED
|
@@ -301,24 +301,61 @@ Worker pool over direct browser — browser processes are isolated in workers so
|
|
|
301
301
|
in one cannot affect the orchestrator or other sessions.
|
|
302
302
|
|
|
303
303
|
Pinned browser stack — `playwright-core` and `impit` are pinned to exact versions and
|
|
304
|
-
`camoufox-js` to its `0.10.x` line
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
`only-allow pnpm` preinstall guard that
|
|
311
|
-
`npm install -g
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
304
|
+
`camoufox-js` to its `0.10.x` line; the three are coupled and upgraded together, because
|
|
305
|
+
each floating range broke fresh consumer installs that our lockfile masked.
|
|
306
|
+
playwright-core stays at `1.60.0` (1.61+ rejects camoufox's Juggler and fails every
|
|
307
|
+
launch — corroborated upstream, since camoufox-js `0.12.0` declares
|
|
308
|
+
`peerDependencies: { "playwright-core": "<1.61.0" }`, the same bound this pin holds by
|
|
309
|
+
hand). `impit` stays at `0.13.0`: camoufox-js 0.10.2 requires `^0.13.0`, and within that
|
|
310
|
+
range only 0.13.0 lacks the `only-allow pnpm` preinstall guard that breaks
|
|
311
|
+
`npm install -g` (an upstream mistake in 0.13.1/0.14.0, dropped again in 0.14.1). An
|
|
312
|
+
exact pin is required because npm `overrides` do not propagate to consumers. Rationale in
|
|
313
|
+
full: `src/infrastructure/browser/thread-worker-browser.ts`.
|
|
314
|
+
|
|
315
|
+
The two original blockers have both lapsed — camoufox restored Windows binaries in
|
|
316
|
+
`v152.0.4-beta.26` (2026-07-16) and impit's guard is long gone — but camoufox-js `0.12.0`
|
|
317
|
+
is held back by a third, worse one: it requires `better-sqlite3 ^13.0.1`, and
|
|
318
|
+
better-sqlite3 13 **dropped prebuilt binaries**. 12.x installs via
|
|
319
|
+
`prebuild-install || node-gyp rebuild`; 13.x ships no install script at all, so npm
|
|
320
|
+
auto-runs `node-gyp rebuild` against its `binding.gyp` and every install demands a full
|
|
321
|
+
C++ toolchain. This was measured, not inferred: the upgrade was made, and `npm ci` failed
|
|
322
|
+
outright on the Windows CI runner (`could not find a version of Visual Studio 2017 or
|
|
323
|
+
newer`) — i.e. it would break installation for most Windows users, which is exactly what
|
|
324
|
+
this pin exists to prevent. Check better-sqlite3 before camoufox next time.
|
|
325
|
+
|
|
326
|
+
The browser BINARY, by contrast, is not pinned and cannot be. `camoufox-js fetch` takes
|
|
327
|
+
no version argument: it walks the `daijro/camoufox` GitHub releases newest-first and takes
|
|
328
|
+
the first non-prerelease release carrying an asset for this OS/arch. So the binary a
|
|
329
|
+
consumer gets is whatever camoufox published most recently at install time, regardless of
|
|
330
|
+
which camoufox-js version is installed — the npm pins do not freeze it, and a future
|
|
331
|
+
camoufox release could break launches for fresh installs with no change on our side.
|
|
332
|
+
Windows assets were in fact missing from `v146-hardware` through `v152.0.2-alpha` and
|
|
333
|
+
returned in `v152.0.4-beta.26` (2026-07-16). Current newest is `v152.0.4-beta.28`
|
|
334
|
+
(Firefox 152); it launches and drives cleanly under playwright-core `1.60.0` **with
|
|
335
|
+
camoufox-js 0.10.2** (whose `MIN_VERSION` is `beta.19`, so it accepts it), verified
|
|
336
|
+
directly, as does the older `v135.0.1-beta.24` an existing cache may still hold. The
|
|
337
|
+
practical consequence is that browser freshness is independent of the npm pin: a stale
|
|
338
|
+
`camoufox-js` does not mean a stale Firefox. Re-verify a real launch when bumping this
|
|
339
|
+
stack — the unit and integration suites mock the browser and cannot catch a Juggler
|
|
340
|
+
mismatch.
|
|
315
341
|
|
|
316
342
|
Pinned data stack — `apache-arrow` is a direct dependency at `21.1.0`, and `overrides`
|
|
317
343
|
forces the whole tree to that single version so LanceDB and Arrow share one Arrow instance
|
|
318
344
|
(mismatched Arrow copies do not interoperate — arrays built by one are rejected by the
|
|
319
|
-
other). This sits above `@lancedb/lancedb` 0.
|
|
320
|
-
(`>=15.0.0 <=18.1.0`)
|
|
321
|
-
whenever `@lancedb/lancedb` is
|
|
345
|
+
other). This sits above `@lancedb/lancedb` 0.37's declared Arrow peer ceiling
|
|
346
|
+
(`>=15.0.0 <=18.1.0`) — npm will not even resolve the pairing without the override — and
|
|
347
|
+
it is verified working, but it should be re-validated whenever `@lancedb/lancedb` is
|
|
348
|
+
upgraded.
|
|
349
|
+
|
|
350
|
+
`21.1.0` is exact for a measured reason, not caution. Bumping the PATCH-looking minor to
|
|
351
|
+
`21.2.0` breaks the store outright: LanceDB's Rust-side Arrow reader cannot parse the
|
|
352
|
+
schema that Arrow 21.2 writes, failing every table open with
|
|
353
|
+
`Failed to read IPC file: Arrow error: Parser error: Unable to get root as footer:
|
|
354
|
+
RangeOutOfBounds … UnionVariant { variant: "Type::FixedSizeList" }` — 56 unit and 36
|
|
355
|
+
integration tests, every one that touches a real table. Do not treat this range as
|
|
356
|
+
caret-safe. Note also that every `@lancedb/lancedb` release through 0.37 declares the same
|
|
357
|
+
`<=18.1.0` Arrow ceiling, so upgrading LanceDB does not resolve the override; it only
|
|
358
|
+
changes which pairing needs re-validating.
|
|
322
359
|
|
|
323
360
|
Pinned validation library — `typebox` is pinned to the exact version the pi host packages
|
|
324
361
|
depend on (`@earendil-works/pi-ai` / `@earendil-works/pi-coding-agent` 0.84.2 pin `1.3.7`). Every
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,72 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.3.12] - 2026-08-15
|
|
9
|
+
|
|
10
|
+
A six-way independent audit of the whole tree (core/orchestration/LLM,
|
|
11
|
+
infrastructure/security, knowledge + scrape pipeline, tools/TUI/CLI,
|
|
12
|
+
release/CI/docs, and test-suite integrity), run against the clean 1.3.11 tree,
|
|
13
|
+
followed by a second pass that exercised each external integration live rather
|
|
14
|
+
than reading it. Every new regression test is mutation-verified — re-run against
|
|
15
|
+
the reinstated pre-fix code and confirmed to fail.
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
|
|
19
|
+
- **`security_search` returned the oldest CVEs on record and could not reach recent ones.** NVD 2.0 returns keyword matches oldest-first and offers no sort parameter (`sortBy` is a 404), so paging from index 0 and stopping at `maxResults` deterministically kept the least useful window. Measured against the live API: a search for `openssl` returned CVE-1999-0428 through 2004 out of 566 matches, and nothing published after 2004 was reachable at any page size — the researcher was being handed 27-year-old advisories as the current vulnerability picture. The client now reads `totalResults` from the first page and seeks once to the tail, then orders the window newest-first (undated entries last). Only fires when the match set is larger than what is returned, so a query whose results already fit still costs exactly one request. The same search now returns the 20 most recent, through 2026-08-13. OSV had a milder form of this — its per-package list is in ID order, so truncation dropped advisories on alphabetical accident — and is now sorted newest-first before the cut.
|
|
20
|
+
- **The NVD rate limiter could leave a search permanently unsettled.** The inter-request spacing timer (up to 6s without an API key) was `unref`'d, but that timer *is* the operation — nothing else settles the awaited promise. With no other ref'd handle, Node treated the process as idle mid-search and the promise never resolved: no result, no error, no timeout. Reproduced deterministically as soon as a second request was issued. It is now ref'd, and remains bounded and abortable so shutdown cannot be delayed beyond one spacing interval. Every other `unref` in the tree guards a timeout against real I/O that holds the loop open by itself; this was the only one with nothing behind it.
|
|
21
|
+
- **ReDoS in the JWT redaction pattern, the sibling the 1.3.9 fix did not reach.** Bounding the URL-credential scheme fixed one quadratic pattern and left an identical one next to it: JWT segments exclude `.`, so an unbounded `[A-Za-z0-9_-]+` consumes to the end of a dotless base64url run and then backtracks one character at a time — and because `_` counts as a non-alphanumeric left boundary, the same run offers a fresh start position every few characters. Measured 393ms of blocked event loop per log message across the 40 000-character scan window, on a function that runs for every log line over largely untrusted content. Bounding the segments brings the identical input to 41ms with no loss of coverage; the bounds sit far above any real token.
|
|
22
|
+
- **`PI_RESEARCH_DISABLED_TOOLS` and the SDK's `excludeTools` accepted unknown tool names silently.** `RESEARCH_TOOL_NAMES` documents itself as the vocabulary all three exclusion surfaces accept, but only `--exclude-tools` validated against it — the other two reached the same blind Set union, where a typo excludes nothing while the user believes the tool is off for a whole billed run. Both now warn at the shared chokepoint, naming the offending value and the valid set. A warning rather than an error there: that path serves a library API and an ambient env var, where aborting a legitimate request over one unrecognized name is the wrong trade. The CLI keeps its fail-fast.
|
|
23
|
+
|
|
24
|
+
- **`PI_RESEARCH_STATE_DIR` pointed one directory short could wipe `config.env` and the knowledge store.** `scripts/cleanup.cjs` decided whether an override was safe to remove wholesale by testing whether `<dirName>/research` appeared as any two adjacent path segments — a naming-convention match, not the ownership proof its own comment required. That also matched the research **root** (`~/.pi/research`, which holds `config.env` and `knowledge_db/`), unrelated directories beneath it (`/mnt/backup/.pi/research/mystate`), and traversal forms (`.../research/state/../../..`) that resolve outside our tree entirely — each taking a recursive `rmSync` with no prompt. Dropping the documented `/state` suffix is a one-directory-off mistake the runtime honors literally, so a deliberate `npm run cleanup` could silently destroy every saved setting and the entire knowledge store. The check now requires the full default shape — the resolved path's trailing segments being `<dirName>/research/state` — which keeps a relocated-but-default layout reclaimable while refusing the root, siblings, and traversal escapes; everything else falls through to the targeted, filename-scoped sweep.
|
|
25
|
+
- **The GPU lock could be stolen from a live process that was still using it.** The staleness backstop was a hardcoded 5 minutes, which exactly equalled the default `EMBEDDING_MODEL_INIT_TIMEOUT_MS` (300000, schema max 600000) — and the embedder holds that lock across the whole model download plus pipeline load. Dead owners are already reclaimed separately via PID+startTime liveness, so this path only ever fires against a process just proven alive: a first-time model fetch running inside its own permitted budget would have the lock taken out from under it, re-admitting exactly the concurrent WebGPU/Dawn initialization the lock exists to prevent. The threshold is now 15 minutes, strictly above the configurable maximum hold.
|
|
26
|
+
- **The CPU-fallback embedder load leaked a native ONNX session on timeout.** `loadPipelineWithTimeout` grew a late-arrival disposal guard in 1.3.11, but the sibling `loadModelOnCPU` — the path taken *after* a WebGPU failure, on a host already under memory pressure — kept the same uncancellable `withTimeout` race with no such guard, orphaning one native session per retry. It now disposes the late arrival identically.
|
|
27
|
+
- **Concurrent knowledge searches tore down each other's TUI indicator.** The widget ID is a single global slot and the tool is `executionMode: 'parallel'` (and separately drivable via `/knowledge-store`), so the first search to finish removed the panel while a sibling was still running. Show/hide is now reference-counted; only the last one out removes it.
|
|
28
|
+
- **`registerAndReplace()` had no disposal-in-flight guard.** Unlike `clear()`/`replace()`/`disposeAll()`, it read and disposed `registration.instance` without joining an in-flight `disposalPromise`, then swapped the whole registration object — so a concurrent teardown could dispose the same instance twice and have its replacement orphaned. Currently unreachable (no call sites), fixed as a latent landmine.
|
|
29
|
+
- **The SSRF-safe undici `Agent` was never released, and was not actually a singleton.** It is meant to be one process-lifetime instance holding keep-alive sockets to every host scraped; the CLI hard-exits so it never noticed, but an embedding host (the pi extension) survives SDK init/shutdown cycles. Two problems: nothing ever closed it, and the cache held the *resolved value*, assigned only after `await import('undici')` — so every concurrent first caller passed the `undefined` check, built its own `Agent`, and all but the last were handed live instances the cache never referenced. Those opened sockets that no shutdown path could ever close, which is the very leak being fixed; scrapes run concurrently by design, so the first burst hit it. The cache now holds the construction *promise*, so concurrent callers join one build, and `shutdownResearchSDK()` closes it and clears the cache — awaiting an in-flight construction rather than orphaning it — so a later re-initialization rebuilds instead of reusing a closed Agent.
|
|
30
|
+
- **A cancelled run reported itself as a component failure in `--json`.** A Ctrl-C unwinding through the planner surfaces internally as `Coordinator failed: Request was aborted`, and that raw wording was emitted as the payload's `error`. The plain-text branch has always refused to say "failed" for a deliberate stop — "Cancellation is a completed intention, not a failure" — but JSON said it anyway, and SKILL.md instructs agents to relay these messages, so a user who pressed Ctrl-C was told the coordinator broke. `error` now reads `pi-research research cancelled.` with the original wording preserved under `cause`; `cancelled`, `exitCode` and the absence of `retryable` are unchanged. Verified against a real interrupted run: exit 130, correct payload.
|
|
31
|
+
- **Two prompts hardcoded the stale-year examples that `injectCurrentDate` was deliberately generalized away from.** That helper derives its examples from the current year precisely so the guidance "stays correct in any year (a hardcoded '2024 or 2025' reads wrong from 2027 on)" — and the coordinator and evaluator templates then said exactly `2024 or 2025` in their own bodies, duplicating the injected guidance with values that expire. From January 2027 they would have told the model the stale years were 2024/2025 while the actually-stale ones were 2025/2026, implicitly legitimising the year to avoid. Both now refer to the injected anchor instead of restating years.
|
|
32
|
+
- **`PLAYWRIGHT_BROWSERS_PATH` moved where the browser was looked for but never where it was installed** — plumbing added, and the documentation corrected to stop overclaiming. It was documented as overriding "where the camoufox binary is stored and looked up" and only ever did the second half: camoufox-js below 0.12.0 hardcodes `userCacheDir("camoufox")` and honours no variable at all. Setting it downloads to the default cache, leaves the postinstall's "already installed" probe permanently unsatisfied (so every install re-fetches), and makes the runtime report the browser missing right after a successful download. `CAMOUFOX_INSTALL_DIR` — the variable camoufox-js 0.12.0 introduced — now takes precedence, `PLAYWRIGHT_BROWSERS_PATH` is an alias, and both the postinstall fetch and the browser workers are handed it. **This is inert on the pinned camoufox-js 0.10.x** (see the dependency note below for why the pin cannot advance), so the plumbing is correct-and-waiting rather than a live fix; `CONFIGURATION.md` now states plainly that a custom browser directory is unsupported until the pin moves.
|
|
33
|
+
|
|
34
|
+
### Tests
|
|
35
|
+
|
|
36
|
+
- The 1.3.11 backup-pruner TTL retention fix — a silent permanent-data-loss fix — shipped with no regression test. Six now cover it (retention inside the TTL, pruning past it, never deleting the newest, ranking by the trailing backup timestamp rather than whole-name sort, migration-temp reaping, and never touching a temp-named live table); two fail against the pre-fix code.
|
|
37
|
+
- `task-execution-service`'s module-level liveness-probe cache (1s) and restart cooldown (10s) both outlive a test, and the reset seam that existed for exactly this had zero callers — leaving the suite order-dependent. The seam now resets both globals and is called in `beforeEach`.
|
|
38
|
+
- `cli-bridge-env.test.ts` wrote fixtures into the real home directory; a hard kill before its cleanup stranded them with nothing to sweep up. It now redirects `HOME`/`USERPROFILE` to a temp dir.
|
|
39
|
+
- Fake timers in `circuit-breaker.test.ts` and `retry-utils.test.ts` were never uninstalled (`restoreAllMocks` does not do it), leaving the clock faked at file end.
|
|
40
|
+
- **`verify-package` now checks that `dist/prompts` is byte-identical to `src/prompts`, not merely present.** The existing check counted files, so a stale copy passed every gate — and `dist/prompts` is what the bundled CLI and the agent-skill engine actually load. Editing a prompt without rebuilding leaves the shipped product running the old instructions with a fully green suite; that state existed in this working tree and the new check caught it. A truncated or partial copy fails here too.
|
|
41
|
+
- **The shipped prompt templates are now rendered through the real substitution path and asserted to leave no `{{placeholder}}` behind**, for the coordinator and for the evaluator at all three complexity levels (its template branches on complexity, so one level passing proves little). Previously one placeholder was covered, by name, against a stub template — so adding a placeholder to a `.md` without wiring it into the call map sent literal braces to the model as instructions it could not act on. The test reads the real files, so it cannot drift from what ships.
|
|
42
|
+
- New coverage for the NVD recency seek (tail offset requested, no extra request when results already fit, newest-first ordering) and for the rate-limiter timer staying ref'd — the last asserted directly, because that failure is invisible under a test runner, which always holds the event loop open by itself.
|
|
43
|
+
- **The JWT ReDoS regression test asserts a complexity class rather than a wall-clock budget.** Its first form used a 200ms threshold with roughly 2× headroom, which cannot work here: the 193-file parallel suite inflated a 41ms post-fix pass to 288ms, overlapping the ~393ms pre-fix cost on an idle machine, so any fixed number either flakes under load or stops catching the regression. It now doubles the input and requires the cost not to more than double — both measurements inflate together under load, so the ratio is stable. Measured 1.87× fixed against 4.01× unbounded on the raw pattern (2.06× through `redactSecrets`), threshold 3.0, mutation-verified at 4.80× against the pre-fix pattern.
|
|
44
|
+
|
|
45
|
+
### Changed
|
|
46
|
+
|
|
47
|
+
- **npm keywords aligned with the repository's GitHub topics.** All eight topics (`agent-skills`, `ai-research`, `browser-automation`, `claude-code`, `llm-tools`, `web-access`, `web-scraping`, `web-search`) are now present; `claude-code` and `codex` were missing entirely despite both being first-class agent-skill install targets alongside the already-listed `openclaw`. Added rather than replaced: `pi-package` is load-bearing, not decorative — pi's own package documentation states the pi.dev gallery lists packages carrying that keyword, so swapping the list wholesale for the GitHub topics would have delisted this package from the pi ecosystem. pi has no separate tag field of its own (its manifest accepts only `extensions`/`skills`/`prompts`/`themes`/`video`/`image`), so npm `keywords` is the single tag surface for both registries.
|
|
48
|
+
|
|
49
|
+
### Added
|
|
50
|
+
|
|
51
|
+
- **`status` now reports whether the stealth browser is installed**, with the exact command to fix it. This became urgent rather than cosmetic: npm 11.19 blocks package install scripts by default, so a fresh `npm install -g @lincoln504/pi-research` now skips this package's `postinstall` — and therefore the camoufox download — while reporting success. Verified by installing the packed tarball under npm 11.19: everything else survives (onnxruntime, sharp, esbuild and webgpu all ship prebuilt platform packages, and `dist/` is in the tarball), so the browser is the single casualty and nothing else signals it. Previously `status` answered `ready: true` with no browser present and the first symptom was a scrape degrading mid-run. A missing browser is reported alongside `ready` rather than folded into it, because the fetch path still serves most pages. The check is a resolve plus an `existsSync` — it never launches anything.
|
|
52
|
+
|
|
53
|
+
### Dependencies
|
|
54
|
+
|
|
55
|
+
Every upgrade below was validated by actually running the subsystem it touches, not by a green suite alone — the browser is mocked throughout the test suite, so a Juggler mismatch would reach production with the whole unit suite passing.
|
|
56
|
+
|
|
57
|
+
- **Browser binary refreshed to Firefox 152 without moving the npm pin.** The camoufox *binary* is fetched dynamically and is not governed by the `camoufox-js` version, so `v152.0.4-beta.28` (Firefox 152) runs fine on the pinned camoufox-js 0.10.2, whose `MIN_VERSION` of `beta.19` accepts it. That closes a real staleness gap — the stealth browser had been presenting Firefox 135 from March 2025 while the fetch path advertised Firefox 153, itself a bot signal. Verified by launching and driving a real browser (both 152.0.4-beta.28 and the older 135.0.1-beta.24 launch, navigate and report a correct UA under playwright-core 1.60.0) and by all 13 live-network search/scrape connectivity tests.
|
|
58
|
+
- **`camoufox-js` stays on ^0.10.2 and `impit` on 0.13.0 — attempted, measured, reverted.** Both blockers ARCHITECTURE.md recorded had genuinely lapsed (camoufox restored Windows assets in `v152.0.4-beta.26`; impit's `only-allow pnpm` guard existed only in 0.13.1/0.14.0), so the upgrade was made — and CI caught a third, worse blocker on Windows. camoufox-js 0.12.0 requires `better-sqlite3 ^13.0.1`, and better-sqlite3 13 **dropped prebuilt binaries**: 12.x installs via `prebuild-install || node-gyp rebuild`, while 13.x ships no install script at all, so npm auto-runs `node-gyp rebuild` against its `binding.gyp` and every install demands a full C++ toolchain. `npm ci` failed outright on the Windows runner (`could not find a version of Visual Studio 2017 or newer`) — i.e. it would break installation for most Windows users, exactly what this pin exists to prevent. The blocker is now better understood than before: check better-sqlite3 first, not camoufox.
|
|
59
|
+
- **`@lancedb/lancedb` 0.29.0 → 0.37.1.** Validated beyond the suite: 0.37.1 opens a real 116 MB, 2249-row store written by 0.29 and serves a live knowledge search against it with real embeddings. Two measured invariants the store's own code depends on were re-checked and hold: the three commit-conflict strings `isLanceCommitConflict` matches are still present in the 0.37.1 native binary, and `createIndex` still advances `version()` by exactly one for a vector index, an FTS index and a `replace: true` rebuild alike.
|
|
60
|
+
- **`youtubei.js` ^17 → ^18** (live transcript fetch verified) and **`@types/jsdom` ^28 → ^30**.
|
|
61
|
+
|
|
62
|
+
Held deliberately, each for a checked reason rather than caution:
|
|
63
|
+
|
|
64
|
+
- **`apache-arrow` stays at `21.1.0`.** The minor bump to 21.2.0 breaks the knowledge store outright — LanceDB's Rust-side reader cannot parse the schema Arrow 21.2 writes, failing every real-table open with `Unable to get root as footer: RangeOutOfBounds … Type::FixedSizeList`, across 56 unit and 36 integration tests. Attempted, measured, reverted; the pin's rationale is now empirical rather than assumed.
|
|
65
|
+
- **`bgutils-js` stays at ^3.** v4 removes the root export entirely (no `"."` in its `exports` map), so the `await import('bgutils-js')` the PoToken path uses would throw outright, and restructures the API (`Challenge.create` → a free `getChallenge`, `BotGuardClient` now event-emitter based with a different `snapshot` shape). That is a rewrite of the most fragile integration in the tree, and v3 demonstrably works — a live transcript fetch succeeds today.
|
|
66
|
+
- **`playwright-core` stays at `1.60.0`** (camoufox-js peer range, above) and **`typebox` at `1.3.7`** (pi 0.84.2, the newest published, pins exactly that).
|
|
67
|
+
- **`typescript` stays in `>=6.0.3 <6.1.0`.** TypeScript 7 is unusable here regardless of its own merits: typescript-eslint's newest release (8.67.0) still declares `typescript: ">=4.8.4 <6.1.0"` and hardcodes the same supported range, so 7.x would break linting. 6.0.3 is already the newest release in the supported line.
|
|
68
|
+
|
|
69
|
+
### Verified, no change needed
|
|
70
|
+
|
|
71
|
+
- Every external integration was exercised against its live API rather than read: NVD, OSV, CISA KEV, and GitHub Advisories all return current data; YouTube transcripts resolve correctly despite `youtubei.js` and `bgutils-js` each being a major behind; StackExchange returns canonical high-score answers on the path the tool actually uses. CISA KEV and GitHub Advisories already order results deliberately (by due date and `direction=desc` respectively) and were left alone.
|
|
72
|
+
- The published package points `pi.extensions` at raw `src/`, which contains two `enum`s and parameter properties — both fatal under Node's strip-only type stripping. pi loads extensions through `jiti`, which transpiles fully, so this is safe; noted because the failure mode would be a total load failure, not a degradation.
|
|
73
|
+
|
|
8
74
|
## [1.3.11] - 2026-08-14
|
|
9
75
|
|
|
10
76
|
Alongside the revert below, a four-way independent audit (docs-vs-code
|
package/docs/CONFIGURATION.md
CHANGED
|
@@ -79,7 +79,7 @@ Research
|
|
|
79
79
|
| `PI_RESEARCH_MAX_CONCURRENT_RUNS` | `3` | ≥1 | Machine-wide cap on research runs executing at the same time, across **every** process (CLI, agent skill, pi extension, SDK). Runs beyond the cap queue rather than fail. All concurrent runs share one leader-elected browser/embedding pool, so oversubscribing it degrades every run at once. |
|
|
80
80
|
| `PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS` | `600000` | ≥0 | How long a run queues for a free slot before failing with "maximum concurrent research runs reached" (CLI exit `75`). `0` = fail immediately instead of queueing. |
|
|
81
81
|
| `PI_RESEARCH_MODEL` | _(pi: session model; CLI/skill: required)_ | — | The model research runs on. **Required for the standalone CLI / agent skill** — they use only this configured model (never the model selected inside the pi extension) and refuse to start without one (the CLI's per-run `--model` flag also satisfies this). On the SDK it selects the session model when no `model` option is given. In the pi extension it overrides researcher sub-agents and knowledge synthesis, while the coordinator and evaluator keep using the session model. Accepts `provider/id` or a bare model id. |
|
|
82
|
-
| `PI_RESEARCH_DISABLED_TOOLS` | _(none)_ | — | Comma-separated research tools to disable for a run (`search`, `scrape`, `security_search`, `stackexchange`, `youtube_transcript`, `grep`). Removed from every researcher's toolset and named in the coordinator/evaluator prompt. Strictly additive — it can only remove capabilities, never grant one, and it stacks on top of the default exclusions rather than replacing them. |
|
|
82
|
+
| `PI_RESEARCH_DISABLED_TOOLS` | _(none)_ | — | Comma-separated research tools to disable for a run (`search`, `scrape`, `security_search`, `stackexchange`, `youtube_transcript`, `grep`, `read`). Removed from every researcher's toolset and named in the coordinator/evaluator prompt. Strictly additive — it can only remove capabilities, never grant one, and it stacks on top of the default exclusions rather than replacing them. An unrecognized name excludes nothing and is warned about rather than failing the run. |
|
|
83
83
|
| `PI_RESEARCH_REPORT_EXPORT_ENABLED` (TUI) | `false` | — | Front-ends write a Markdown report to disk and surface its path. |
|
|
84
84
|
| `PI_RESEARCH_REPORT_EXPORT_DIR` | _(smart cwd)_ | — | Pin exported reports to a fixed directory, bypassing the cwd-relative resolution. Useful for the agent skill, which runs from the host agent's arbitrary directory. |
|
|
85
85
|
| `PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING` | `0.15` | 0.05–1.0 | Max fraction of the context window used for initial scrape context. |
|
|
@@ -156,7 +156,8 @@ Diagnostics & platform
|
|
|
156
156
|
| `PI_RESEARCH_SKILL_DIR` | _(auto)_ | Override the bundled research-skill source directory used by the skill installer. |
|
|
157
157
|
| `PI_RESEARCH_PURGE_BROWSERS` | _(unset)_ | Read by the bundled `scripts/cleanup.cjs`: set `1` to also delete the shared camoufox browser cache (kept by default because other installs may use it). Note npm ≥7 does not run `preuninstall`, so that script does not fire on `npm uninstall` — see [AGENT-SKILL.md](AGENT-SKILL.md). |
|
|
158
158
|
| `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` | _(unset)_ | Set `1` during `npm install` to skip the camoufox browser download (fetched lazily on first use instead; standard Playwright convention). |
|
|
159
|
-
| `
|
|
159
|
+
| `CAMOUFOX_INSTALL_DIR` | _(user cache)_ | camoufox-js's own variable, and the only one that can relocate the browser. Sets where pi-research looks the binary up, and is exported to the postinstall fetch and to browser workers. **Not effective on the currently pinned camoufox-js (0.10.x), which hardcodes its cache directory and honours no variable** — so relocating the browser is unsupported until that pin can advance (see ARCHITECTURE.md). |
|
|
160
|
+
| `PLAYWRIGHT_BROWSERS_PATH` | _(user cache)_ | Alias for the above, accepted for compatibility and exported onward as `CAMOUFOX_INSTALL_DIR`. Same caveat: on the pinned camoufox-js it moves only where pi-research looks, not where the download lands, so setting it reports the browser missing from the custom path after a successful install. |
|
|
160
161
|
| `XDG_CACHE_HOME` | `~/.cache` | Standard XDG variable. When set, every `~/.cache/pi-research/...` path below is rooted at `$XDG_CACHE_HOME/pi-research/...` instead. |
|
|
161
162
|
| `PI_RESEARCH_BIN` (alias `PI_RESEARCH_PATH`) | _(auto)_ | Agent-skill launcher only: explicit path to the pi-research engine binary when auto-resolution (PATH → local install → npx) should be bypassed. See [skills/pi-research/references/configuration.md](../skills/pi-research/references/configuration.md). |
|
|
162
163
|
| `PLAYWRIGHT_INSTALL_DEPS` | _(unset)_ | Linux only. Set `true` during `npm install` to also install system libraries via `npx playwright install-deps` (same as `npm run install:system-deps`). |
|