@lincoln504/pi-research 1.3.4 → 1.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/cli.mjs +67 -25
- package/dist/thread-worker.mjs +69 -16
- package/docs/CHANGELOG.md +37 -0
- package/package.json +1 -1
- package/skills/pi-research/SKILL.md +1 -1
- package/src/infrastructure/browser/browser-client.ts +5 -2
- package/src/infrastructure/browser/thread-worker-browser.ts +29 -0
- package/src/infrastructure/browser/thread-worker-messaging.ts +45 -8
- package/src/infrastructure/browser/thread-worker.mjs +69 -16
- package/src/security/osv-types.ts +1 -1
- package/src/skill-install/skill-installer.ts +29 -1
- package/src/tools/research-tool-definition.ts +3 -2
- package/src/tools/scrape.ts +6 -4
- package/src/tools/security.ts +13 -4
- package/src/utils/log-rotation.ts +21 -6
- package/src/utils/log-utils.ts +13 -2
- package/src/utils/text-utils.ts +6 -2
- package/src/youtube/potoken.ts +26 -2
- package/src/youtube/transcript-client.ts +5 -2
package/README.md
CHANGED
|
@@ -43,7 +43,7 @@ ask in natural language and the tool picks the right one:
|
|
|
43
43
|
|
|
44
44
|
### Use cases
|
|
45
45
|
|
|
46
|
-
- Plug-and-play (no API key needed) research tool for your [
|
|
46
|
+
- Plug-and-play (no API key needed) research tool for your [Pi](https://github.com/earendil-works/pi) Coding Agent.
|
|
47
47
|
- Research from Claude Code, Codex, or another coding agent, with a cheaper lightweight or local model driving the research so it doesn't spend your main agent's budget.
|
|
48
48
|
- Populating a dataset or building an index of knowledge sources from the web.
|
|
49
49
|
- Holding research in the knowledge store with a configurable scope — project-specific or globally user-scoped, set per directory from the `/research-config` TUI.
|
|
@@ -91,3 +91,8 @@ and the stealth-browser stack. v1.0.0 is the first stable release since. Channel
|
|
|
91
91
|
|
|
92
92
|
MIT. Bundled third-party licenses are listed in
|
|
93
93
|
[docs/THIRD-PARTY-NOTICES.md](docs/THIRD-PARTY-NOTICES.md).
|
|
94
|
+
|
|
95
|
+
### Package identity
|
|
96
|
+
|
|
97
|
+
This project is always `@lincoln504/pi-research` on npm. The unscoped `pi-research`
|
|
98
|
+
package is unrelated and deprecated.
|
package/dist/cli.mjs
CHANGED
|
@@ -215,7 +215,8 @@ function formatArg(arg) {
|
|
|
215
215
|
return String(arg);
|
|
216
216
|
}
|
|
217
217
|
function redactSecrets(message) {
|
|
218
|
-
|
|
218
|
+
const rawLength = message.length;
|
|
219
|
+
let out = rawLength > REDACT_SCAN_LENGTH ? message.slice(0, REDACT_SCAN_LENGTH) : message;
|
|
219
220
|
out = out.replace(ANSI_PATTERN, "");
|
|
220
221
|
out = out.replace(URL_CREDENTIALS_PATTERN, "$1[REDACTED]@");
|
|
221
222
|
out = out.replace(JWT_PATTERN, "[REDACTED]");
|
|
@@ -232,14 +233,14 @@ function redactSecrets(message) {
|
|
|
232
233
|
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
233
234
|
}
|
|
234
235
|
if (out.length > MAX_LOG_MESSAGE_LENGTH) {
|
|
235
|
-
out = `${out.slice(0, keep)}\u2026[truncated ${
|
|
236
|
+
out = `${out.slice(0, keep)}\u2026[truncated ${rawLength - keep} chars]`;
|
|
236
237
|
}
|
|
237
238
|
return out;
|
|
238
239
|
}
|
|
239
240
|
function stripTerminalEscapes(message) {
|
|
240
241
|
return message.replace(TERMINAL_ESCAPE_PATTERN, "").replace(CONTROL_CHARS_PATTERN, " ");
|
|
241
242
|
}
|
|
242
|
-
var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, TERMINAL_ESCAPE_PATTERN;
|
|
243
|
+
var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, REDACT_SCAN_LENGTH, TERMINAL_ESCAPE_PATTERN;
|
|
243
244
|
var init_log_utils = __esm({
|
|
244
245
|
"src/utils/log-utils.ts"() {
|
|
245
246
|
"use strict";
|
|
@@ -256,6 +257,7 @@ var init_log_utils = __esm({
|
|
|
256
257
|
PROVIDER_TOKEN_PATTERN = /\b(AIza[0-9A-Za-z_-]{35}|ya29\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g;
|
|
257
258
|
LONG_HEX_SECRET_PATTERN = /\b[0-9a-fA-F]{32,}\b/g;
|
|
258
259
|
CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
|
|
260
|
+
REDACT_SCAN_LENGTH = MAX_LOG_MESSAGE_LENGTH * 4;
|
|
259
261
|
TERMINAL_ESCAPE_PATTERN = new RegExp(
|
|
260
262
|
"\\x1b\\[[0-?]*[ -/]*[@-~]?|\\x1b\\][^\\x07\\x1b]*(?:\\x07|\\x1b\\\\)?|\\x1b[PX^_][^\\x1b]*(?:\\x1b\\\\)?|\\x1b.?",
|
|
261
263
|
"g"
|
|
@@ -266,10 +268,17 @@ var init_log_utils = __esm({
|
|
|
266
268
|
// src/utils/log-rotation.ts
|
|
267
269
|
import * as fs from "node:fs";
|
|
268
270
|
import * as path2 from "node:path";
|
|
269
|
-
|
|
271
|
+
function touchOwnerOnly(filePath, mode) {
|
|
272
|
+
try {
|
|
273
|
+
fs.closeSync(fs.openSync(filePath, APPEND_CREATE_NOFOLLOW_FLAGS, mode));
|
|
274
|
+
} catch {
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
var APPEND_CREATE_NOFOLLOW_FLAGS, LogRotation;
|
|
270
278
|
var init_log_rotation = __esm({
|
|
271
279
|
"src/utils/log-rotation.ts"() {
|
|
272
280
|
"use strict";
|
|
281
|
+
APPEND_CREATE_NOFOLLOW_FLAGS = fs.constants.O_WRONLY | fs.constants.O_APPEND | fs.constants.O_CREAT | (typeof fs.constants.O_NOFOLLOW === "number" ? fs.constants.O_NOFOLLOW : 0);
|
|
273
282
|
LogRotation = class {
|
|
274
283
|
MAX_LOG_SIZE = 10 * 1024 * 1024;
|
|
275
284
|
// 10MB max file size
|
|
@@ -287,10 +296,7 @@ var init_log_rotation = __esm({
|
|
|
287
296
|
try {
|
|
288
297
|
if (fs.existsSync(logFile)) {
|
|
289
298
|
fs.unlinkSync(logFile);
|
|
290
|
-
|
|
291
|
-
fs.closeSync(fs.openSync(logFile, "a", 384));
|
|
292
|
-
} catch {
|
|
293
|
-
}
|
|
299
|
+
touchOwnerOnly(logFile, 384);
|
|
294
300
|
try {
|
|
295
301
|
fs.chmodSync(logFile, 384);
|
|
296
302
|
} catch {
|
|
@@ -330,10 +336,7 @@ var init_log_rotation = __esm({
|
|
|
330
336
|
this.logger.log("[Logger] Log rotation skipped (rename failed; file likely in use):", renameErr);
|
|
331
337
|
return;
|
|
332
338
|
}
|
|
333
|
-
|
|
334
|
-
fs.closeSync(fs.openSync(logFile, "a", 384));
|
|
335
|
-
} catch {
|
|
336
|
-
}
|
|
339
|
+
touchOwnerOnly(logFile, 384);
|
|
337
340
|
try {
|
|
338
341
|
fs.chmodSync(logFile, 384);
|
|
339
342
|
} catch {
|
|
@@ -9988,11 +9991,31 @@ function reconcileSkillInstalls(opts = {}) {
|
|
|
9988
9991
|
if (source !== null) {
|
|
9989
9992
|
try {
|
|
9990
9993
|
fs16.unlinkSync(e.path);
|
|
9994
|
+
} catch {
|
|
9995
|
+
keep.push(e);
|
|
9996
|
+
continue;
|
|
9997
|
+
}
|
|
9998
|
+
try {
|
|
9991
9999
|
fs16.symlinkSync(source, e.path, process.platform === "win32" ? "junction" : "dir");
|
|
9992
10000
|
keep.push({ ...e, type: "symlink", source });
|
|
9993
10001
|
result.repointed.push(e.path);
|
|
9994
10002
|
} catch {
|
|
9995
|
-
|
|
10003
|
+
const dir = path17.dirname(e.path);
|
|
10004
|
+
const base = path17.basename(e.path);
|
|
10005
|
+
const suffix = crypto10.randomBytes(4).toString("hex");
|
|
10006
|
+
const fresh = path17.join(dir, `.${base}.new-${suffix}`);
|
|
10007
|
+
try {
|
|
10008
|
+
copyDir(source, fresh);
|
|
10009
|
+
fs16.renameSync(fresh, e.path);
|
|
10010
|
+
keep.push({ ...e, type: "copy", source });
|
|
10011
|
+
result.repointed.push(e.path);
|
|
10012
|
+
} catch {
|
|
10013
|
+
try {
|
|
10014
|
+
fs16.rmSync(fresh, { recursive: true, force: true });
|
|
10015
|
+
} catch {
|
|
10016
|
+
}
|
|
10017
|
+
result.pruned.push(e.path);
|
|
10018
|
+
}
|
|
9996
10019
|
}
|
|
9997
10020
|
} else if (!targetLive) {
|
|
9998
10021
|
try {
|
|
@@ -11439,6 +11462,7 @@ function isFullMockMode() {
|
|
|
11439
11462
|
init_config();
|
|
11440
11463
|
init_logger();
|
|
11441
11464
|
init_error_tracker();
|
|
11465
|
+
init_log_utils();
|
|
11442
11466
|
init_http_body();
|
|
11443
11467
|
import * as http2 from "node:http";
|
|
11444
11468
|
|
|
@@ -11574,7 +11598,7 @@ var BrowserClient = class {
|
|
|
11574
11598
|
resolve9(parsed);
|
|
11575
11599
|
}
|
|
11576
11600
|
} catch (_e) {
|
|
11577
|
-
const raw = collected.toString();
|
|
11601
|
+
const raw = redactSecrets(collected.toString());
|
|
11578
11602
|
const preview = raw.length > 200 ? raw.slice(0, 200) + "..." : raw;
|
|
11579
11603
|
const error = new Error(`Failed to parse response (status ${res.statusCode}): ${preview}`);
|
|
11580
11604
|
errorTracker.trackError(error, {
|
|
@@ -17545,6 +17569,7 @@ async function scrape(urls, maxConcurrency = 5, signal, config, sessionId, onUrl
|
|
|
17545
17569
|
// src/tools/scrape.ts
|
|
17546
17570
|
init_constants();
|
|
17547
17571
|
init_text_utils();
|
|
17572
|
+
init_log_utils();
|
|
17548
17573
|
init_config();
|
|
17549
17574
|
init_service_registry();
|
|
17550
17575
|
init_service_interfaces();
|
|
@@ -17716,18 +17741,19 @@ ${evictedDuplicates.map((u) => `- ${u}`).join("\n")}
|
|
|
17716
17741
|
let freshResults;
|
|
17717
17742
|
const concurrency = p.maxConcurrency || defaultConcurrency;
|
|
17718
17743
|
try {
|
|
17719
|
-
const results = await scrape(urlsToFetch, concurrency, signal,
|
|
17744
|
+
const results = await scrape(urlsToFetch, concurrency, signal, config, getGlobalState().researchId, (result) => {
|
|
17720
17745
|
options.onUrlScrapeResult?.(result.url, result.success);
|
|
17721
17746
|
}, container);
|
|
17722
17747
|
freshResults = Array.isArray(results) ? results : [];
|
|
17723
17748
|
} catch (error) {
|
|
17724
|
-
|
|
17749
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
17750
|
+
logger.error(`[scrape tool] Scrape failed: ${message}`);
|
|
17725
17751
|
metrics.increment("tool_scrape_calls_total", 1, { status: "error" });
|
|
17726
17752
|
return {
|
|
17727
17753
|
content: [{ type: "text", text: `# Scrape Failed
|
|
17728
17754
|
|
|
17729
|
-
${
|
|
17730
|
-
details: { error: String(error) }
|
|
17755
|
+
${redactSecrets(message)}` }],
|
|
17756
|
+
details: { error: redactSecrets(String(error)) }
|
|
17731
17757
|
};
|
|
17732
17758
|
}
|
|
17733
17759
|
const successfulFresh = freshResults.filter((r) => r.success);
|
|
@@ -18974,7 +19000,7 @@ function mapOsvItemToVulnerability(item) {
|
|
|
18974
19000
|
for (const cwe of item.database_specific.cwe) {
|
|
18975
19001
|
if (typeof cwe === "string") {
|
|
18976
19002
|
cwes.push(cwe);
|
|
18977
|
-
} else if (typeof cwe === "object" && typeof cwe.id === "string") {
|
|
19003
|
+
} else if (typeof cwe === "object" && cwe !== null && typeof cwe.id === "string") {
|
|
18978
19004
|
cwes.push(cwe.id);
|
|
18979
19005
|
}
|
|
18980
19006
|
}
|
|
@@ -19355,6 +19381,10 @@ async function searchSecurityDatabases(params, signal) {
|
|
|
19355
19381
|
// src/tools/security.ts
|
|
19356
19382
|
init_constants();
|
|
19357
19383
|
init_metrics();
|
|
19384
|
+
var MAX_DESCRIPTION_CHARS = 300;
|
|
19385
|
+
function truncateDescription(description) {
|
|
19386
|
+
return description.length > MAX_DESCRIPTION_CHARS ? `${description.substring(0, MAX_DESCRIPTION_CHARS)}...` : description;
|
|
19387
|
+
}
|
|
19358
19388
|
function createSecuritySearchTool(options) {
|
|
19359
19389
|
const SecuritySearchParamsSchema = Type6.Object({
|
|
19360
19390
|
databases: Type6.Optional(Type6.Array(Type6.String({
|
|
@@ -19557,7 +19587,7 @@ Unable to search security databases. This may be a temporary issue - try again l
|
|
|
19557
19587
|
}
|
|
19558
19588
|
}
|
|
19559
19589
|
const description = vuln.description;
|
|
19560
|
-
markdown += `- **Description:** ${description
|
|
19590
|
+
markdown += `- **Description:** ${truncateDescription(description)}
|
|
19561
19591
|
`;
|
|
19562
19592
|
if (vuln.knownExploited === true) {
|
|
19563
19593
|
markdown += "- **[Actively Exploited]**\n";
|
|
@@ -19609,7 +19639,7 @@ Unable to search security databases. This may be a temporary issue - try again l
|
|
|
19609
19639
|
`;
|
|
19610
19640
|
}
|
|
19611
19641
|
const description = vuln.description;
|
|
19612
|
-
markdown += `- **Description:** ${description
|
|
19642
|
+
markdown += `- **Description:** ${truncateDescription(description)}
|
|
19613
19643
|
`;
|
|
19614
19644
|
if (vuln.dueDate !== void 0) {
|
|
19615
19645
|
markdown += `- **Due Date:** ${vuln.dueDate}
|
|
@@ -19657,7 +19687,7 @@ Unable to search security databases. This may be a temporary issue - try again l
|
|
|
19657
19687
|
markdown += `- **Summary:** ${adv.summary}
|
|
19658
19688
|
`;
|
|
19659
19689
|
const description = adv.description ?? "";
|
|
19660
|
-
markdown += `- **Description:** ${description
|
|
19690
|
+
markdown += `- **Description:** ${truncateDescription(description)}
|
|
19661
19691
|
`;
|
|
19662
19692
|
if (adv.published) {
|
|
19663
19693
|
markdown += `- **Published:** ${adv.published}
|
|
@@ -19699,7 +19729,7 @@ Unable to search security databases. This may be a temporary issue - try again l
|
|
|
19699
19729
|
markdown += `- **Severity:** ${vuln.severity}
|
|
19700
19730
|
`;
|
|
19701
19731
|
const description = vuln.description;
|
|
19702
|
-
markdown += `- **Description:** ${description
|
|
19732
|
+
markdown += `- **Description:** ${truncateDescription(description)}
|
|
19703
19733
|
`;
|
|
19704
19734
|
if (vuln.affectedProducts && vuln.affectedProducts.length > 0) {
|
|
19705
19735
|
markdown += `- **Affected:** ${vuln.affectedProducts.join(", ")}
|
|
@@ -20628,11 +20658,13 @@ function watchUrl(videoId) {
|
|
|
20628
20658
|
init_retry_utils();
|
|
20629
20659
|
init_circuit_breaker();
|
|
20630
20660
|
init_logger();
|
|
20661
|
+
init_log_utils();
|
|
20631
20662
|
init_metrics();
|
|
20632
20663
|
init_http_body();
|
|
20633
20664
|
|
|
20634
20665
|
// src/youtube/potoken.ts
|
|
20635
20666
|
init_logger();
|
|
20667
|
+
init_safe_unref();
|
|
20636
20668
|
init_metrics();
|
|
20637
20669
|
var DEFAULT_REQUEST_KEY = "O43z0dpjhgX20SCx4KAo";
|
|
20638
20670
|
var BRIDGED_GLOBALS = ["window", "document", "location", "navigator", "origin"];
|
|
@@ -20645,9 +20677,19 @@ function describeError(error) {
|
|
|
20645
20677
|
return String(error);
|
|
20646
20678
|
}
|
|
20647
20679
|
var mintChain = Promise.resolve();
|
|
20680
|
+
var MINT_HARD_RELEASE_MS = 5 * 60 * 1e3;
|
|
20648
20681
|
async function mintPoTokens(identifiers, opts = {}) {
|
|
20649
20682
|
const run = mintChain.then(() => doMint(identifiers, opts), () => doMint(identifiers, opts));
|
|
20650
|
-
|
|
20683
|
+
const settled = run.then(() => void 0, () => void 0);
|
|
20684
|
+
let hardTimer;
|
|
20685
|
+
const hardRelease = new Promise((resolve9) => {
|
|
20686
|
+
hardTimer = setTimeout(() => {
|
|
20687
|
+
logger.warn("[youtube] potoken mint mutex force-released \u2014 doMint() did not settle within the hard timeout");
|
|
20688
|
+
resolve9();
|
|
20689
|
+
}, MINT_HARD_RELEASE_MS);
|
|
20690
|
+
safeUnref(hardTimer);
|
|
20691
|
+
});
|
|
20692
|
+
mintChain = Promise.race([settled, hardRelease]).finally(() => clearTimeout(hardTimer));
|
|
20651
20693
|
return run;
|
|
20652
20694
|
}
|
|
20653
20695
|
async function doMint(identifiers, opts) {
|
|
@@ -20929,7 +20971,7 @@ function parseJson3(body) {
|
|
|
20929
20971
|
try {
|
|
20930
20972
|
parsed = JSON.parse(body);
|
|
20931
20973
|
} catch {
|
|
20932
|
-
logger.debug(`[youtube] timedtext body was not valid json3 (first 120 chars): ${body.slice(0, 120)}`);
|
|
20974
|
+
logger.debug(`[youtube] timedtext body was not valid json3 (first 120 chars): ${redactSecrets(body).slice(0, 120)}`);
|
|
20933
20975
|
return "";
|
|
20934
20976
|
}
|
|
20935
20977
|
const events = parsed.events;
|
package/dist/thread-worker.mjs
CHANGED
|
@@ -45,7 +45,8 @@ function formatArg(arg) {
|
|
|
45
45
|
return String(arg);
|
|
46
46
|
}
|
|
47
47
|
function redactSecrets(message) {
|
|
48
|
-
|
|
48
|
+
const rawLength = message.length;
|
|
49
|
+
let out = rawLength > REDACT_SCAN_LENGTH ? message.slice(0, REDACT_SCAN_LENGTH) : message;
|
|
49
50
|
out = out.replace(ANSI_PATTERN, "");
|
|
50
51
|
out = out.replace(URL_CREDENTIALS_PATTERN, "$1[REDACTED]@");
|
|
51
52
|
out = out.replace(JWT_PATTERN, "[REDACTED]");
|
|
@@ -62,14 +63,14 @@ function redactSecrets(message) {
|
|
|
62
63
|
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
63
64
|
}
|
|
64
65
|
if (out.length > MAX_LOG_MESSAGE_LENGTH) {
|
|
65
|
-
out = `${out.slice(0, keep)}\u2026[truncated ${
|
|
66
|
+
out = `${out.slice(0, keep)}\u2026[truncated ${rawLength - keep} chars]`;
|
|
66
67
|
}
|
|
67
68
|
return out;
|
|
68
69
|
}
|
|
69
70
|
function stripTerminalEscapes(message) {
|
|
70
71
|
return message.replace(TERMINAL_ESCAPE_PATTERN, "").replace(CONTROL_CHARS_PATTERN, " ");
|
|
71
72
|
}
|
|
72
|
-
var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, TERMINAL_ESCAPE_PATTERN;
|
|
73
|
+
var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, REDACT_SCAN_LENGTH, TERMINAL_ESCAPE_PATTERN;
|
|
73
74
|
var init_log_utils = __esm({
|
|
74
75
|
"src/utils/log-utils.ts"() {
|
|
75
76
|
"use strict";
|
|
@@ -86,6 +87,7 @@ var init_log_utils = __esm({
|
|
|
86
87
|
PROVIDER_TOKEN_PATTERN = /\b(AIza[0-9A-Za-z_-]{35}|ya29\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g;
|
|
87
88
|
LONG_HEX_SECRET_PATTERN = /\b[0-9a-fA-F]{32,}\b/g;
|
|
88
89
|
CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
|
|
90
|
+
REDACT_SCAN_LENGTH = MAX_LOG_MESSAGE_LENGTH * 4;
|
|
89
91
|
TERMINAL_ESCAPE_PATTERN = new RegExp(
|
|
90
92
|
"\\x1b\\[[0-?]*[ -/]*[@-~]?|\\x1b\\][^\\x07\\x1b]*(?:\\x07|\\x1b\\\\)?|\\x1b[PX^_][^\\x1b]*(?:\\x1b\\\\)?|\\x1b.?",
|
|
91
93
|
"g"
|
|
@@ -357,10 +359,17 @@ var init_error_tracker = __esm({
|
|
|
357
359
|
// src/utils/log-rotation.ts
|
|
358
360
|
import * as fs2 from "node:fs";
|
|
359
361
|
import * as path2 from "node:path";
|
|
360
|
-
|
|
362
|
+
function touchOwnerOnly(filePath, mode) {
|
|
363
|
+
try {
|
|
364
|
+
fs2.closeSync(fs2.openSync(filePath, APPEND_CREATE_NOFOLLOW_FLAGS, mode));
|
|
365
|
+
} catch {
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
var APPEND_CREATE_NOFOLLOW_FLAGS, LogRotation;
|
|
361
369
|
var init_log_rotation = __esm({
|
|
362
370
|
"src/utils/log-rotation.ts"() {
|
|
363
371
|
"use strict";
|
|
372
|
+
APPEND_CREATE_NOFOLLOW_FLAGS = fs2.constants.O_WRONLY | fs2.constants.O_APPEND | fs2.constants.O_CREAT | (typeof fs2.constants.O_NOFOLLOW === "number" ? fs2.constants.O_NOFOLLOW : 0);
|
|
364
373
|
LogRotation = class {
|
|
365
374
|
MAX_LOG_SIZE = 10 * 1024 * 1024;
|
|
366
375
|
// 10MB max file size
|
|
@@ -378,10 +387,7 @@ var init_log_rotation = __esm({
|
|
|
378
387
|
try {
|
|
379
388
|
if (fs2.existsSync(logFile)) {
|
|
380
389
|
fs2.unlinkSync(logFile);
|
|
381
|
-
|
|
382
|
-
fs2.closeSync(fs2.openSync(logFile, "a", 384));
|
|
383
|
-
} catch {
|
|
384
|
-
}
|
|
390
|
+
touchOwnerOnly(logFile, 384);
|
|
385
391
|
try {
|
|
386
392
|
fs2.chmodSync(logFile, 384);
|
|
387
393
|
} catch {
|
|
@@ -421,10 +427,7 @@ var init_log_rotation = __esm({
|
|
|
421
427
|
this.logger.log("[Logger] Log rotation skipped (rename failed; file likely in use):", renameErr);
|
|
422
428
|
return;
|
|
423
429
|
}
|
|
424
|
-
|
|
425
|
-
fs2.closeSync(fs2.openSync(logFile, "a", 384));
|
|
426
|
-
} catch {
|
|
427
|
-
}
|
|
430
|
+
touchOwnerOnly(logFile, 384);
|
|
428
431
|
try {
|
|
429
432
|
fs2.chmodSync(logFile, 384);
|
|
430
433
|
} catch {
|
|
@@ -1921,6 +1924,7 @@ init_scraper_types();
|
|
|
1921
1924
|
import { appendFileSync as appendFileSync2 } from "node:fs";
|
|
1922
1925
|
var workerId2 = "";
|
|
1923
1926
|
var pageCreationLock = Promise.resolve();
|
|
1927
|
+
var PAGE_CREATION_HARD_RELEASE_MS = 5 * 60 * 1e3;
|
|
1924
1928
|
async function createPageSafe(context2) {
|
|
1925
1929
|
let release;
|
|
1926
1930
|
const nextLock = new Promise((resolve) => {
|
|
@@ -1930,7 +1934,18 @@ async function createPageSafe(context2) {
|
|
|
1930
1934
|
pageCreationLock = currentLock.then(() => nextLock);
|
|
1931
1935
|
await currentLock;
|
|
1932
1936
|
const pagePromise = context2.newPage();
|
|
1933
|
-
|
|
1937
|
+
let hardReleased = false;
|
|
1938
|
+
const releaseOnce = () => {
|
|
1939
|
+
if (hardReleased) return;
|
|
1940
|
+
hardReleased = true;
|
|
1941
|
+
clearTimeout(hardReleaseTimer);
|
|
1942
|
+
release();
|
|
1943
|
+
};
|
|
1944
|
+
const hardReleaseTimer = setTimeout(() => {
|
|
1945
|
+
logToDebugFile2("WARN", "[ThreadWorker] page-creation mutex force-released \u2014 newPage() did not settle within the hard timeout");
|
|
1946
|
+
releaseOnce();
|
|
1947
|
+
}, PAGE_CREATION_HARD_RELEASE_MS);
|
|
1948
|
+
pagePromise.then(releaseOnce, releaseOnce);
|
|
1934
1949
|
pagePromise.catch((err) => logToDebugFile2("DEBUG", `[ThreadWorker] Background page creation rejection: ${err.message}`));
|
|
1935
1950
|
let timeoutId;
|
|
1936
1951
|
try {
|
|
@@ -1957,6 +1972,15 @@ async function createPageSafe(context2) {
|
|
|
1957
1972
|
function setWorkerId2(id) {
|
|
1958
1973
|
workerId2 = id;
|
|
1959
1974
|
}
|
|
1975
|
+
function parseTimeoutMs(envVar, def) {
|
|
1976
|
+
if (envVar === void 0 || envVar.trim() === "") return def;
|
|
1977
|
+
const n = Number(envVar.trim());
|
|
1978
|
+
if (!Number.isFinite(n) || n < 0) {
|
|
1979
|
+
logToDebugFile2("WARN", `[ThreadWorker] Invalid timeout value "${envVar}", using default: ${def}ms`);
|
|
1980
|
+
return def;
|
|
1981
|
+
}
|
|
1982
|
+
return n;
|
|
1983
|
+
}
|
|
1960
1984
|
function logToDebugFile2(level, ...args) {
|
|
1961
1985
|
const logFile = process.env["PI_RESEARCH_LOG_FILE"];
|
|
1962
1986
|
if (!logFile) return;
|
|
@@ -2066,7 +2090,7 @@ async function extractSearchResults(page) {
|
|
|
2066
2090
|
}
|
|
2067
2091
|
async function executeSearchTask(_context, query) {
|
|
2068
2092
|
const page = await createPageSafe(_context);
|
|
2069
|
-
const SEARCH_TIMEOUT =
|
|
2093
|
+
const SEARCH_TIMEOUT = parseTimeoutMs(process.env["PI_RESEARCH_SEARCH_TIMEOUT_MS"], 45e3);
|
|
2070
2094
|
page.setDefaultTimeout(SEARCH_TIMEOUT);
|
|
2071
2095
|
page.setDefaultNavigationTimeout(SEARCH_TIMEOUT);
|
|
2072
2096
|
try {
|
|
@@ -2135,7 +2159,7 @@ function seedSubresourceVerdict(validatedUrl) {
|
|
|
2135
2159
|
}
|
|
2136
2160
|
async function executeScrapeTask(_context, url, signal) {
|
|
2137
2161
|
const page = await createPageSafe(_context);
|
|
2138
|
-
const SCRAPE_TIMEOUT =
|
|
2162
|
+
const SCRAPE_TIMEOUT = parseTimeoutMs(process.env["PI_RESEARCH_SCRAPE_TIMEOUT_MS"], 15e3);
|
|
2139
2163
|
page.setDefaultTimeout(SCRAPE_TIMEOUT);
|
|
2140
2164
|
page.setDefaultNavigationTimeout(SCRAPE_TIMEOUT);
|
|
2141
2165
|
const onAbort = () => {
|
|
@@ -2301,7 +2325,7 @@ async function executeHealthCheckAttempt(_context, navTimeoutMs, url = HEALTH_PR
|
|
|
2301
2325
|
}
|
|
2302
2326
|
}
|
|
2303
2327
|
async function executeHealthCheck(_context, initMs) {
|
|
2304
|
-
const configuredMs =
|
|
2328
|
+
const configuredMs = parseTimeoutMs(process.env["PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS"], 0);
|
|
2305
2329
|
const HEALTH_TIMEOUT = configuredMs > 0 ? Math.max(1e4, configuredMs) : 1e4;
|
|
2306
2330
|
if (initMs > 3e3) {
|
|
2307
2331
|
logToDebugFile2("WARN", `[Worker-${workerId2}] Browser init was slow: ${initMs}ms \u2014 leaving less headroom for DDG navigation.`);
|
|
@@ -2594,6 +2618,35 @@ async function initBrowser() {
|
|
|
2594
2618
|
height: 1080,
|
|
2595
2619
|
colorDepth: 24,
|
|
2596
2620
|
pixelRatio: 1
|
|
2621
|
+
},
|
|
2622
|
+
// Mute all audio output at the engine level. The general scrape
|
|
2623
|
+
// path drives a REAL Firefox (Camoufox), and a researcher can be
|
|
2624
|
+
// pointed at a youtube.com watch page through the ordinary
|
|
2625
|
+
// `scrape` tool (as opposed to the dedicated `youtube_transcript`
|
|
2626
|
+
// tool, which never touches the browser) — YouTube watch pages
|
|
2627
|
+
// autoplay video with sound, so without an explicit mute the
|
|
2628
|
+
// process plays audible audio through the host's sound device.
|
|
2629
|
+
//
|
|
2630
|
+
// media.volume_scale forces the browser's effective output
|
|
2631
|
+
// volume to 0 regardless of per-tab mute/autoplay state or any
|
|
2632
|
+
// page script re-enabling audio — the most reliable, engine-level
|
|
2633
|
+
// mute available for headless Firefox/Camoufox (the same pref
|
|
2634
|
+
// Selenium/Playwright Firefox test grids use to silence CI runs).
|
|
2635
|
+
// media.autoplay.default=5 additionally blocks autoplay outright
|
|
2636
|
+
// (belt-and-suspenders; some sites route around the volume scale
|
|
2637
|
+
// via Web Audio gain nodes). Neither pref affects rendering — the
|
|
2638
|
+
// page still loads and paints normally, only the audio output
|
|
2639
|
+
// device is silenced.
|
|
2640
|
+
//
|
|
2641
|
+
// Set on every platform/headless-mode combination, not just
|
|
2642
|
+
// Linux: this was only *observed* on Linux (true headless and
|
|
2643
|
+
// Xvfb-backed 'virtual' mode both retain a real PulseAudio/
|
|
2644
|
+
// PipeWire/ALSA sink for the Firefox process to play through),
|
|
2645
|
+
// but headless Firefox on macOS/Windows is not guaranteed to be
|
|
2646
|
+
// silent either, so the mute is applied unconditionally.
|
|
2647
|
+
firefox_user_prefs: {
|
|
2648
|
+
"media.volume_scale": "0.0",
|
|
2649
|
+
"media.autoplay.default": 5
|
|
2597
2650
|
}
|
|
2598
2651
|
});
|
|
2599
2652
|
launchPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser launch rejection: ${err.message}`));
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,43 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.3.5] - 2026-08-10
|
|
9
|
+
|
|
10
|
+
Two rounds of workflow-backed code review on the clean 1.3.4 tree (high-effort,
|
|
11
|
+
then a wider max-effort pass explicitly told not to re-find the first round's
|
|
12
|
+
results), followed by a triage of every open CodeQL alert. Every fix below is
|
|
13
|
+
covered by a passing regression test; two bugs (the OSV null-CWE crash and the
|
|
14
|
+
duration-formatter rounding bug) had no prior test coverage at all, so tests
|
|
15
|
+
were added alongside the fixes.
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
|
|
19
|
+
- **The browser page-creation mutex could still hang forever.** 1.3.4 fixed the mutex releasing too early (before `newPage()` itself settled). That fix had no bound of its own: if `newPage()` never settles at all — not just slow, but genuinely wedged — the lock is now held forever, livelocking every future `createPageSafe()` call on that worker thread. Added a 5-minute hard-release timer that force-releases the lock if `newPage()` still hasn't settled, independent of the settlement-based release for the normal case.
|
|
20
|
+
- **`reconcileSkillInstalls()`'s stale-symlink re-point had no copy fallback.** The sibling `installSkill()` already falls back to a copy when `symlinkSync` fails after the old link is removed (a Windows cross-volume junction failure, most commonly). The periodic reconciler's structurally identical re-point lacked that fallback, so a `symlinkSync` failure there silently left the manifest claiming a skill was installed after the file was already deleted from disk. Added the same fallback — and staged the copy into a dot-prefixed temp dir with an atomic rename into place, so a copy that fails partway (disk full, interrupted process) can never leave an unrecognized, SKILL.md-less partial directory orphaned at the skill's path.
|
|
21
|
+
- **The scrape tool's own gating used a different config than its actual fetch.** `createScrapeTool()` resolves `config = options.config || getConfig(options.ctx.cwd)` and uses that resolved value for every batching/gating decision (knowledge-store mode, concurrency caps, token-fraction limits, pool capacity) — except the real `scrape()` call, which passed the raw, possibly-`undefined` `options.config`. A caller that omits `config` (the parameter is typed optional) got gating computed for one cwd while the actual network fetch silently fell back to whatever `getConfig()` resolves for `process.cwd()` — a different worker/pool concurrency or provider config than what the visible gating decision assumed, with nothing logged to indicate the mismatch.
|
|
22
|
+
- **`redactSecrets()`'s truncate-before-redact fix (1.3.4) removed its own cost bound.** Moving truncation after the full regex chain (correctly, to stop a secret leaking at the truncation boundary) also removed the implicit cap that used to limit those ~10 sequential regex passes to `MAX_LOG_MESSAGE_LENGTH` characters regardless of input size. A verbose/debug log call carrying a multi-megabyte scraped payload now ran the whole chain across the untruncated string. Pre-slices to 4× `MAX_LOG_MESSAGE_LENGTH` before redacting — generous enough that no in-bounds secret is ever missed (patterns only shrink text, never expand it, and content beyond that window can never survive into the final truncated output anyway) — and reports the truncation count against the true raw length instead of the post-redaction length.
|
|
23
|
+
- **Six more truncate-before-redact / unredacted-error-text sites, the same bug class 1.3.4 fixed at five others.** `scrape.ts`'s and `research-tool-definition.ts`'s error-catch paths returned raw `error.message`/`String(error)` directly as tool output; `browser-client.ts`'s JSON-parse-failure handler embedded a raw 200-char response preview in the thrown `Error`; `transcript-client.ts`'s debug log sliced YouTube's raw `timedtext` response to 120 chars before redaction instead of after. Each of these can carry a credential (a failing URL's userinfo, an echoed `Authorization` header, an embedded token in a malformed page) into a log file, a thrown error, or the agent's own conversation/report output. All six now redact before truncating or returning.
|
|
24
|
+
- **A `null` entry in an OSV/GHSA advisory's `database_specific.cwe` crashed the entire lookup.** `typeof cwe === 'object'` is `true` for `null`, so `mapOsvItemToVulnerability` threw on `(null).id` — and since it runs uncaught inside `osv-client.ts`'s per-term fetch (only wrapped in `Promise.allSettled`), one malformed record silently discarded every vulnerability found for that search term, replaced by a generic failure note. Added the missing `cwe !== null` check.
|
|
25
|
+
- **The YouTube PoToken mint mutex had the same unbounded-hang exposure as the browser mutex above.** `doMint()` awaits BotGuard VM execution and interpreter eval against a hostile/malformed attestation script from Google, neither of which take an abort signal — if either hangs instead of rejecting, the module-level `mintChain` mutex never settles, and every subsequent `mintPoTokens()` call across all researchers queues behind it forever. Added the same style of hard-release bound as the browser mutex fix, via a `Promise.race` against a 5-minute (unref'd) timeout.
|
|
26
|
+
- **Three timeout env vars silently became near-zero on a malformed value.** `PI_RESEARCH_SEARCH_TIMEOUT_MS`, `PI_RESEARCH_SCRAPE_TIMEOUT_MS`, and `PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS` were read with `parseInt()`, which truncates at the first non-digit instead of rejecting the value — `parseInt('1m', 10)` is `1`, not `NaN`, so a plausible duration-shorthand typo silently became a 1ms browser timeout instead of falling back to the default, failing every search/scrape/health-check on that worker for the rest of its life with no warning. Replaced with a shared helper using `Number()` (which rejects trailing garbage) plus a warning log on an invalid value.
|
|
27
|
+
- **`formatDuration()` could render the nonsensical "1m 60s".** Rounding `remainingSeconds` independently of `minutes` let a value like 119,601ms (1m 59.601s) round its seconds component up to 60 without rolling into the next minute. Now rounds the total seconds first and derives minutes/seconds from that integer.
|
|
28
|
+
- **A theoretical symlink race on log rotation's file reopen.** After renaming the live log aside during rotation (or unlinking it during `clearLogs`), the code reopens the same path with `fs.openSync(path, 'a', 0o600)` to recreate it owner-only — a window in which a local attacker with write access to the log directory could plant a symlink at that path, causing the reopen to follow it and (re)create/chmod an arbitrary file with this process's permissions. Added `O_NOFOLLOW` to the reopen (a no-op on Windows, where the flag isn't defined) so it refuses to follow a symlink instead.
|
|
29
|
+
- **Scraping a YouTube watch page could play audible audio through the host's speakers.** The general `scrape` tool drives a real stealth Firefox (Camoufox) — unlike the dedicated `youtube_transcript` tool, which only calls YouTube's API (`youtubei.js`/`timedtext`) and never touches a browser. A researcher agent scraping a `youtube.com` watch-page URL through the ordinary `scrape` tool hit YouTube's default video autoplay with no engine-level mute configured, so the browser process played sound through whatever audio sink it could reach. Reported on Linux, where both true-headless and the Xvfb-backed `'virtual'` mode keep a live PulseAudio/PipeWire/ALSA sink reachable — but headless Firefox is not guaranteed silent on macOS/Windows either, so the fix (`media.volume_scale: '0.0'` plus `media.autoplay.default: 5` in the shared Camoufox launch options) is unconditional across platforms and headless modes, not Linux-specific.
|
|
30
|
+
|
|
31
|
+
### Reviewed, not changed
|
|
32
|
+
|
|
33
|
+
A full CodeQL alert triage found 12 of 13 open alerts to be false positives or out-of-scope for a code fix, each verified individually rather than dismissed on assumption:
|
|
34
|
+
- Two `js/missing-await` alerts in `embedder.ts` are reference-identity checks (`this.recoveryPromise === recovery`) misread as a forgotten `await` — awaiting either side would break the logic.
|
|
35
|
+
- A `js/file-system-race` alert in `file-lock-service.ts` is on code that already reopens with the `'wx'` exclusive-create flag specifically to make the check-then-open atomic.
|
|
36
|
+
- A `js/insecure-temporary-file` alert in `browser/cleanup-utils.ts` is a read-only liveness probe on a Firefox lock file, not a file creation.
|
|
37
|
+
- Six alerts (`js/file-system-race`, `js/insecure-temporary-file`) are all in test files creating throwaway, randomly-named fixtures under `os.tmpdir()` for the test's own private use.
|
|
38
|
+
- One (`js/unused-local-variable`) was a genuine unused test import; removed.
|
|
39
|
+
|
|
40
|
+
Patch, not minor: every fix is an internal correctness/security/robustness
|
|
41
|
+
change with no change to pi-research's own CLI, tool, or SDK contract —
|
|
42
|
+
matching this project's established precedent for audit-round releases
|
|
43
|
+
(1.3.1 through 1.3.4).
|
|
44
|
+
|
|
8
45
|
## [1.3.4] - 2026-08-09
|
|
9
46
|
|
|
10
47
|
Two rounds on the clean 1.3.3 tree: a full-tree re-audit (10 independent
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: pi-research
|
|
3
3
|
description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
|
|
4
4
|
license: MIT
|
|
5
|
-
metadata: { "author": "Lincoln Deen", "version": "1.3.
|
|
5
|
+
metadata: { "author": "Lincoln Deen", "version": "1.3.5", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
|
|
6
6
|
allowed-tools: Bash(node:*)
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -12,6 +12,7 @@ import type { SearchResult } from '../../web-research/types.ts';
|
|
|
12
12
|
import type { IScheduler } from '../../core/interfaces/scheduler-interfaces.ts';
|
|
13
13
|
import { logger } from '../../logger.ts';
|
|
14
14
|
import { errorTracker } from '../../utils/error-tracker.ts';
|
|
15
|
+
import { redactSecrets } from '../../utils/log-utils.ts';
|
|
15
16
|
import { Utf8Body } from '../../utils/http-body.ts';
|
|
16
17
|
import { getClientAgent } from './client-agent.ts';
|
|
17
18
|
import type { NodeError } from '../../types/index.ts';
|
|
@@ -200,8 +201,10 @@ export class BrowserClient implements IScheduler {
|
|
|
200
201
|
resolve(parsed);
|
|
201
202
|
}
|
|
202
203
|
} catch (_e) {
|
|
203
|
-
// FIX (#23): Truncate response body in error to prevent data leakage
|
|
204
|
-
|
|
204
|
+
// FIX (#23): Truncate response body in error to prevent data leakage.
|
|
205
|
+
// Redact BEFORE truncating — slicing first can cut a secret-shaped
|
|
206
|
+
// token mid-way, leaving a fragment too short to be recognized.
|
|
207
|
+
const raw = redactSecrets(collected.toString());
|
|
205
208
|
const preview = raw.length > 200 ? raw.slice(0, 200) + '...' : raw;
|
|
206
209
|
const error = new Error(`Failed to parse response (status ${res.statusCode}): ${preview}`);
|
|
207
210
|
errorTracker.trackError(error, {
|
|
@@ -193,6 +193,35 @@ export async function initBrowser(): Promise<void> {
|
|
|
193
193
|
colorDepth: 24,
|
|
194
194
|
pixelRatio: 1,
|
|
195
195
|
},
|
|
196
|
+
// Mute all audio output at the engine level. The general scrape
|
|
197
|
+
// path drives a REAL Firefox (Camoufox), and a researcher can be
|
|
198
|
+
// pointed at a youtube.com watch page through the ordinary
|
|
199
|
+
// `scrape` tool (as opposed to the dedicated `youtube_transcript`
|
|
200
|
+
// tool, which never touches the browser) — YouTube watch pages
|
|
201
|
+
// autoplay video with sound, so without an explicit mute the
|
|
202
|
+
// process plays audible audio through the host's sound device.
|
|
203
|
+
//
|
|
204
|
+
// media.volume_scale forces the browser's effective output
|
|
205
|
+
// volume to 0 regardless of per-tab mute/autoplay state or any
|
|
206
|
+
// page script re-enabling audio — the most reliable, engine-level
|
|
207
|
+
// mute available for headless Firefox/Camoufox (the same pref
|
|
208
|
+
// Selenium/Playwright Firefox test grids use to silence CI runs).
|
|
209
|
+
// media.autoplay.default=5 additionally blocks autoplay outright
|
|
210
|
+
// (belt-and-suspenders; some sites route around the volume scale
|
|
211
|
+
// via Web Audio gain nodes). Neither pref affects rendering — the
|
|
212
|
+
// page still loads and paints normally, only the audio output
|
|
213
|
+
// device is silenced.
|
|
214
|
+
//
|
|
215
|
+
// Set on every platform/headless-mode combination, not just
|
|
216
|
+
// Linux: this was only *observed* on Linux (true headless and
|
|
217
|
+
// Xvfb-backed 'virtual' mode both retain a real PulseAudio/
|
|
218
|
+
// PipeWire/ALSA sink for the Firefox process to play through),
|
|
219
|
+
// but headless Firefox on macOS/Windows is not guaranteed to be
|
|
220
|
+
// silent either, so the mute is applied unconditionally.
|
|
221
|
+
firefox_user_prefs: {
|
|
222
|
+
'media.volume_scale': '0.0',
|
|
223
|
+
'media.autoplay.default': 5,
|
|
224
|
+
},
|
|
196
225
|
});
|
|
197
226
|
// Guard with a hard timeout. We catch the launchPromise rejection so it
|
|
198
227
|
// doesn't surface as an UnhandledPromiseRejection if it rejects after the
|
|
@@ -16,6 +16,11 @@ import { MAX_HTML_SIZE, MAX_PDF_SIZE } from '../../web-research/scraper-types.ts
|
|
|
16
16
|
*/
|
|
17
17
|
let pageCreationLock = Promise.resolve();
|
|
18
18
|
|
|
19
|
+
/** How long createPageSafe() waits for a straggling newPage() before force-releasing
|
|
20
|
+
* the mutex. Must stay well above the 60s soft timeout below — it exists only to
|
|
21
|
+
* bound the pathological case where newPage() never settles at all (see comment below). */
|
|
22
|
+
const PAGE_CREATION_HARD_RELEASE_MS = 5 * 60 * 1000;
|
|
23
|
+
|
|
19
24
|
/** Exported for unit testing of the page-creation mutex's timeout behavior. */
|
|
20
25
|
export async function createPageSafe(context: any): Promise<any> {
|
|
21
26
|
let release!: () => void;
|
|
@@ -25,16 +30,32 @@ export async function createPageSafe(context: any): Promise<any> {
|
|
|
25
30
|
|
|
26
31
|
await currentLock;
|
|
27
32
|
const pagePromise = context.newPage();
|
|
28
|
-
// Release the mutex
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
33
|
+
// Release the mutex once newPage() itself has actually settled — NOT when the
|
|
34
|
+
// timeout race below settles. Releasing in an outer `finally` (the old shape)
|
|
35
|
+
// fired as soon as the 60s timeout won the race, while newPage() was still
|
|
36
|
+
// pending in the background — so the NEXT queued caller started its OWN
|
|
32
37
|
// newPage() concurrently with this still-unsettled one, exactly the concurrent-
|
|
33
38
|
// newPage() scenario this lock exists to prevent (see the comment above), and
|
|
34
39
|
// it happens precisely when the browser is already under the stress that
|
|
35
40
|
// causes 60s+ page-creation stalls in the first place. Attached before the
|
|
36
41
|
// race starts so it's independent of which branch below returns/throws.
|
|
37
|
-
|
|
42
|
+
//
|
|
43
|
+
// A hard release timer bounds the case where newPage() never settles at all
|
|
44
|
+
// (rather than just being slow) — without it, a single wedged newPage() call
|
|
45
|
+
// would hold this lock forever and livelock every future createPageSafe() call
|
|
46
|
+
// on this worker thread.
|
|
47
|
+
let hardReleased = false;
|
|
48
|
+
const releaseOnce = (): void => {
|
|
49
|
+
if (hardReleased) return;
|
|
50
|
+
hardReleased = true;
|
|
51
|
+
clearTimeout(hardReleaseTimer);
|
|
52
|
+
release();
|
|
53
|
+
};
|
|
54
|
+
const hardReleaseTimer = setTimeout(() => {
|
|
55
|
+
logToDebugFile('WARN', '[ThreadWorker] page-creation mutex force-released — newPage() did not settle within the hard timeout');
|
|
56
|
+
releaseOnce();
|
|
57
|
+
}, PAGE_CREATION_HARD_RELEASE_MS);
|
|
58
|
+
pagePromise.then(releaseOnce, releaseOnce);
|
|
38
59
|
pagePromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background page creation rejection: ${err.message}`));
|
|
39
60
|
|
|
40
61
|
let timeoutId: NodeJS.Timeout | undefined;
|
|
@@ -63,6 +84,22 @@ export function setWorkerId(id: string): void {
|
|
|
63
84
|
workerId = id;
|
|
64
85
|
}
|
|
65
86
|
|
|
87
|
+
/**
|
|
88
|
+
* Parse a millisecond timeout from an env var. Uses Number() rather than
|
|
89
|
+
* parseInt() so trailing garbage is REJECTED rather than silently truncated —
|
|
90
|
+
* parseInt('1m', 10) === 1, which turned a plausible duration-shorthand typo
|
|
91
|
+
* into a near-instant timeout instead of falling back to the default.
|
|
92
|
+
*/
|
|
93
|
+
function parseTimeoutMs(envVar: string | undefined, def: number): number {
|
|
94
|
+
if (envVar === undefined || envVar.trim() === '') return def;
|
|
95
|
+
const n = Number(envVar.trim());
|
|
96
|
+
if (!Number.isFinite(n) || n < 0) {
|
|
97
|
+
logToDebugFile('WARN', `[ThreadWorker] Invalid timeout value "${envVar}", using default: ${def}ms`);
|
|
98
|
+
return def;
|
|
99
|
+
}
|
|
100
|
+
return n;
|
|
101
|
+
}
|
|
102
|
+
|
|
66
103
|
/**
|
|
67
104
|
* Log to debug file
|
|
68
105
|
*/
|
|
@@ -205,7 +242,7 @@ export async function executeSearchTask(
|
|
|
205
242
|
query: string
|
|
206
243
|
): Promise<{ results: any[]; jitter: number }> {
|
|
207
244
|
const page = await createPageSafe(_context);
|
|
208
|
-
const SEARCH_TIMEOUT =
|
|
245
|
+
const SEARCH_TIMEOUT = parseTimeoutMs(process.env['PI_RESEARCH_SEARCH_TIMEOUT_MS'], 45000);
|
|
209
246
|
page.setDefaultTimeout(SEARCH_TIMEOUT);
|
|
210
247
|
page.setDefaultNavigationTimeout(SEARCH_TIMEOUT);
|
|
211
248
|
|
|
@@ -322,7 +359,7 @@ export async function executeScrapeTask(
|
|
|
322
359
|
signal?: AbortSignal
|
|
323
360
|
): Promise<{ contentType: string; html?: string; bufferB64?: string; jitter: number }> {
|
|
324
361
|
const page = await createPageSafe(_context);
|
|
325
|
-
const SCRAPE_TIMEOUT =
|
|
362
|
+
const SCRAPE_TIMEOUT = parseTimeoutMs(process.env['PI_RESEARCH_SCRAPE_TIMEOUT_MS'], 15000);
|
|
326
363
|
page.setDefaultTimeout(SCRAPE_TIMEOUT);
|
|
327
364
|
page.setDefaultNavigationTimeout(SCRAPE_TIMEOUT);
|
|
328
365
|
|
|
@@ -666,7 +703,7 @@ export async function executeHealthCheck(
|
|
|
666
703
|
): Promise<{ success: boolean; navMs: number }> {
|
|
667
704
|
// Nav timeout: read from env (passed through getBrowserEnv), floor at 10s.
|
|
668
705
|
// The outer BrowserTaskScheduler.runHealthCheck() has its own 45s hard deadline.
|
|
669
|
-
const configuredMs =
|
|
706
|
+
const configuredMs = parseTimeoutMs(process.env['PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS'], 0);
|
|
670
707
|
const HEALTH_TIMEOUT = configuredMs > 0 ? Math.max(10000, configuredMs) : 10000;
|
|
671
708
|
|
|
672
709
|
if (initMs > 3000) {
|
|
@@ -45,7 +45,8 @@ function formatArg(arg) {
|
|
|
45
45
|
return String(arg);
|
|
46
46
|
}
|
|
47
47
|
function redactSecrets(message) {
|
|
48
|
-
|
|
48
|
+
const rawLength = message.length;
|
|
49
|
+
let out = rawLength > REDACT_SCAN_LENGTH ? message.slice(0, REDACT_SCAN_LENGTH) : message;
|
|
49
50
|
out = out.replace(ANSI_PATTERN, "");
|
|
50
51
|
out = out.replace(URL_CREDENTIALS_PATTERN, "$1[REDACTED]@");
|
|
51
52
|
out = out.replace(JWT_PATTERN, "[REDACTED]");
|
|
@@ -62,14 +63,14 @@ function redactSecrets(message) {
|
|
|
62
63
|
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
63
64
|
}
|
|
64
65
|
if (out.length > MAX_LOG_MESSAGE_LENGTH) {
|
|
65
|
-
out = `${out.slice(0, keep)}\u2026[truncated ${
|
|
66
|
+
out = `${out.slice(0, keep)}\u2026[truncated ${rawLength - keep} chars]`;
|
|
66
67
|
}
|
|
67
68
|
return out;
|
|
68
69
|
}
|
|
69
70
|
function stripTerminalEscapes(message) {
|
|
70
71
|
return message.replace(TERMINAL_ESCAPE_PATTERN, "").replace(CONTROL_CHARS_PATTERN, " ");
|
|
71
72
|
}
|
|
72
|
-
var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, TERMINAL_ESCAPE_PATTERN;
|
|
73
|
+
var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, REDACT_SCAN_LENGTH, TERMINAL_ESCAPE_PATTERN;
|
|
73
74
|
var init_log_utils = __esm({
|
|
74
75
|
"src/utils/log-utils.ts"() {
|
|
75
76
|
"use strict";
|
|
@@ -86,6 +87,7 @@ var init_log_utils = __esm({
|
|
|
86
87
|
PROVIDER_TOKEN_PATTERN = /\b(AIza[0-9A-Za-z_-]{35}|ya29\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g;
|
|
87
88
|
LONG_HEX_SECRET_PATTERN = /\b[0-9a-fA-F]{32,}\b/g;
|
|
88
89
|
CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
|
|
90
|
+
REDACT_SCAN_LENGTH = MAX_LOG_MESSAGE_LENGTH * 4;
|
|
89
91
|
TERMINAL_ESCAPE_PATTERN = new RegExp(
|
|
90
92
|
"\\x1b\\[[0-?]*[ -/]*[@-~]?|\\x1b\\][^\\x07\\x1b]*(?:\\x07|\\x1b\\\\)?|\\x1b[PX^_][^\\x1b]*(?:\\x1b\\\\)?|\\x1b.?",
|
|
91
93
|
"g"
|
|
@@ -357,10 +359,17 @@ var init_error_tracker = __esm({
|
|
|
357
359
|
// src/utils/log-rotation.ts
|
|
358
360
|
import * as fs2 from "node:fs";
|
|
359
361
|
import * as path2 from "node:path";
|
|
360
|
-
|
|
362
|
+
function touchOwnerOnly(filePath, mode) {
|
|
363
|
+
try {
|
|
364
|
+
fs2.closeSync(fs2.openSync(filePath, APPEND_CREATE_NOFOLLOW_FLAGS, mode));
|
|
365
|
+
} catch {
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
var APPEND_CREATE_NOFOLLOW_FLAGS, LogRotation;
|
|
361
369
|
var init_log_rotation = __esm({
|
|
362
370
|
"src/utils/log-rotation.ts"() {
|
|
363
371
|
"use strict";
|
|
372
|
+
APPEND_CREATE_NOFOLLOW_FLAGS = fs2.constants.O_WRONLY | fs2.constants.O_APPEND | fs2.constants.O_CREAT | (typeof fs2.constants.O_NOFOLLOW === "number" ? fs2.constants.O_NOFOLLOW : 0);
|
|
364
373
|
LogRotation = class {
|
|
365
374
|
MAX_LOG_SIZE = 10 * 1024 * 1024;
|
|
366
375
|
// 10MB max file size
|
|
@@ -378,10 +387,7 @@ var init_log_rotation = __esm({
|
|
|
378
387
|
try {
|
|
379
388
|
if (fs2.existsSync(logFile)) {
|
|
380
389
|
fs2.unlinkSync(logFile);
|
|
381
|
-
|
|
382
|
-
fs2.closeSync(fs2.openSync(logFile, "a", 384));
|
|
383
|
-
} catch {
|
|
384
|
-
}
|
|
390
|
+
touchOwnerOnly(logFile, 384);
|
|
385
391
|
try {
|
|
386
392
|
fs2.chmodSync(logFile, 384);
|
|
387
393
|
} catch {
|
|
@@ -421,10 +427,7 @@ var init_log_rotation = __esm({
|
|
|
421
427
|
this.logger.log("[Logger] Log rotation skipped (rename failed; file likely in use):", renameErr);
|
|
422
428
|
return;
|
|
423
429
|
}
|
|
424
|
-
|
|
425
|
-
fs2.closeSync(fs2.openSync(logFile, "a", 384));
|
|
426
|
-
} catch {
|
|
427
|
-
}
|
|
430
|
+
touchOwnerOnly(logFile, 384);
|
|
428
431
|
try {
|
|
429
432
|
fs2.chmodSync(logFile, 384);
|
|
430
433
|
} catch {
|
|
@@ -1921,6 +1924,7 @@ init_scraper_types();
|
|
|
1921
1924
|
import { appendFileSync as appendFileSync2 } from "node:fs";
|
|
1922
1925
|
var workerId2 = "";
|
|
1923
1926
|
var pageCreationLock = Promise.resolve();
|
|
1927
|
+
var PAGE_CREATION_HARD_RELEASE_MS = 5 * 60 * 1e3;
|
|
1924
1928
|
async function createPageSafe(context2) {
|
|
1925
1929
|
let release;
|
|
1926
1930
|
const nextLock = new Promise((resolve) => {
|
|
@@ -1930,7 +1934,18 @@ async function createPageSafe(context2) {
|
|
|
1930
1934
|
pageCreationLock = currentLock.then(() => nextLock);
|
|
1931
1935
|
await currentLock;
|
|
1932
1936
|
const pagePromise = context2.newPage();
|
|
1933
|
-
|
|
1937
|
+
let hardReleased = false;
|
|
1938
|
+
const releaseOnce = () => {
|
|
1939
|
+
if (hardReleased) return;
|
|
1940
|
+
hardReleased = true;
|
|
1941
|
+
clearTimeout(hardReleaseTimer);
|
|
1942
|
+
release();
|
|
1943
|
+
};
|
|
1944
|
+
const hardReleaseTimer = setTimeout(() => {
|
|
1945
|
+
logToDebugFile2("WARN", "[ThreadWorker] page-creation mutex force-released \u2014 newPage() did not settle within the hard timeout");
|
|
1946
|
+
releaseOnce();
|
|
1947
|
+
}, PAGE_CREATION_HARD_RELEASE_MS);
|
|
1948
|
+
pagePromise.then(releaseOnce, releaseOnce);
|
|
1934
1949
|
pagePromise.catch((err) => logToDebugFile2("DEBUG", `[ThreadWorker] Background page creation rejection: ${err.message}`));
|
|
1935
1950
|
let timeoutId;
|
|
1936
1951
|
try {
|
|
@@ -1957,6 +1972,15 @@ async function createPageSafe(context2) {
|
|
|
1957
1972
|
function setWorkerId2(id) {
|
|
1958
1973
|
workerId2 = id;
|
|
1959
1974
|
}
|
|
1975
|
+
function parseTimeoutMs(envVar, def) {
|
|
1976
|
+
if (envVar === void 0 || envVar.trim() === "") return def;
|
|
1977
|
+
const n = Number(envVar.trim());
|
|
1978
|
+
if (!Number.isFinite(n) || n < 0) {
|
|
1979
|
+
logToDebugFile2("WARN", `[ThreadWorker] Invalid timeout value "${envVar}", using default: ${def}ms`);
|
|
1980
|
+
return def;
|
|
1981
|
+
}
|
|
1982
|
+
return n;
|
|
1983
|
+
}
|
|
1960
1984
|
function logToDebugFile2(level, ...args) {
|
|
1961
1985
|
const logFile = process.env["PI_RESEARCH_LOG_FILE"];
|
|
1962
1986
|
if (!logFile) return;
|
|
@@ -2066,7 +2090,7 @@ async function extractSearchResults(page) {
|
|
|
2066
2090
|
}
|
|
2067
2091
|
async function executeSearchTask(_context, query) {
|
|
2068
2092
|
const page = await createPageSafe(_context);
|
|
2069
|
-
const SEARCH_TIMEOUT =
|
|
2093
|
+
const SEARCH_TIMEOUT = parseTimeoutMs(process.env["PI_RESEARCH_SEARCH_TIMEOUT_MS"], 45e3);
|
|
2070
2094
|
page.setDefaultTimeout(SEARCH_TIMEOUT);
|
|
2071
2095
|
page.setDefaultNavigationTimeout(SEARCH_TIMEOUT);
|
|
2072
2096
|
try {
|
|
@@ -2135,7 +2159,7 @@ function seedSubresourceVerdict(validatedUrl) {
|
|
|
2135
2159
|
}
|
|
2136
2160
|
async function executeScrapeTask(_context, url, signal) {
|
|
2137
2161
|
const page = await createPageSafe(_context);
|
|
2138
|
-
const SCRAPE_TIMEOUT =
|
|
2162
|
+
const SCRAPE_TIMEOUT = parseTimeoutMs(process.env["PI_RESEARCH_SCRAPE_TIMEOUT_MS"], 15e3);
|
|
2139
2163
|
page.setDefaultTimeout(SCRAPE_TIMEOUT);
|
|
2140
2164
|
page.setDefaultNavigationTimeout(SCRAPE_TIMEOUT);
|
|
2141
2165
|
const onAbort = () => {
|
|
@@ -2301,7 +2325,7 @@ async function executeHealthCheckAttempt(_context, navTimeoutMs, url = HEALTH_PR
|
|
|
2301
2325
|
}
|
|
2302
2326
|
}
|
|
2303
2327
|
async function executeHealthCheck(_context, initMs) {
|
|
2304
|
-
const configuredMs =
|
|
2328
|
+
const configuredMs = parseTimeoutMs(process.env["PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS"], 0);
|
|
2305
2329
|
const HEALTH_TIMEOUT = configuredMs > 0 ? Math.max(1e4, configuredMs) : 1e4;
|
|
2306
2330
|
if (initMs > 3e3) {
|
|
2307
2331
|
logToDebugFile2("WARN", `[Worker-${workerId2}] Browser init was slow: ${initMs}ms \u2014 leaving less headroom for DDG navigation.`);
|
|
@@ -2594,6 +2618,35 @@ async function initBrowser() {
|
|
|
2594
2618
|
height: 1080,
|
|
2595
2619
|
colorDepth: 24,
|
|
2596
2620
|
pixelRatio: 1
|
|
2621
|
+
},
|
|
2622
|
+
// Mute all audio output at the engine level. The general scrape
|
|
2623
|
+
// path drives a REAL Firefox (Camoufox), and a researcher can be
|
|
2624
|
+
// pointed at a youtube.com watch page through the ordinary
|
|
2625
|
+
// `scrape` tool (as opposed to the dedicated `youtube_transcript`
|
|
2626
|
+
// tool, which never touches the browser) — YouTube watch pages
|
|
2627
|
+
// autoplay video with sound, so without an explicit mute the
|
|
2628
|
+
// process plays audible audio through the host's sound device.
|
|
2629
|
+
//
|
|
2630
|
+
// media.volume_scale forces the browser's effective output
|
|
2631
|
+
// volume to 0 regardless of per-tab mute/autoplay state or any
|
|
2632
|
+
// page script re-enabling audio — the most reliable, engine-level
|
|
2633
|
+
// mute available for headless Firefox/Camoufox (the same pref
|
|
2634
|
+
// Selenium/Playwright Firefox test grids use to silence CI runs).
|
|
2635
|
+
// media.autoplay.default=5 additionally blocks autoplay outright
|
|
2636
|
+
// (belt-and-suspenders; some sites route around the volume scale
|
|
2637
|
+
// via Web Audio gain nodes). Neither pref affects rendering — the
|
|
2638
|
+
// page still loads and paints normally, only the audio output
|
|
2639
|
+
// device is silenced.
|
|
2640
|
+
//
|
|
2641
|
+
// Set on every platform/headless-mode combination, not just
|
|
2642
|
+
// Linux: this was only *observed* on Linux (true headless and
|
|
2643
|
+
// Xvfb-backed 'virtual' mode both retain a real PulseAudio/
|
|
2644
|
+
// PipeWire/ALSA sink for the Firefox process to play through),
|
|
2645
|
+
// but headless Firefox on macOS/Windows is not guaranteed to be
|
|
2646
|
+
// silent either, so the mute is applied unconditionally.
|
|
2647
|
+
firefox_user_prefs: {
|
|
2648
|
+
"media.volume_scale": "0.0",
|
|
2649
|
+
"media.autoplay.default": 5
|
|
2597
2650
|
}
|
|
2598
2651
|
});
|
|
2599
2652
|
launchPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser launch rejection: ${err.message}`));
|
|
@@ -237,7 +237,7 @@ export function mapOsvItemToVulnerability(item: OsvVulnerability): Vulnerability
|
|
|
237
237
|
for (const cwe of item.database_specific.cwe) {
|
|
238
238
|
if (typeof cwe === 'string') {
|
|
239
239
|
cwes.push(cwe);
|
|
240
|
-
} else if (typeof cwe === 'object' && typeof (cwe as { id?: string }).id === 'string') {
|
|
240
|
+
} else if (typeof cwe === 'object' && cwe !== null && typeof (cwe as { id?: string }).id === 'string') {
|
|
241
241
|
cwes.push((cwe as { id: string }).id);
|
|
242
242
|
}
|
|
243
243
|
}
|
|
@@ -647,10 +647,38 @@ export function reconcileSkillInstalls(opts: InstallOptions = {}): ReconcileResu
|
|
|
647
647
|
// Re-point to the current package source (handles update/reinstall).
|
|
648
648
|
try {
|
|
649
649
|
fs.unlinkSync(e.path);
|
|
650
|
+
} catch { keep.push(e); continue; /* old link untouched; retry next startup */ }
|
|
651
|
+
try {
|
|
650
652
|
fs.symlinkSync(source, e.path, process.platform === 'win32' ? 'junction' : 'dir');
|
|
651
653
|
keep.push({ ...e, type: 'symlink', source });
|
|
652
654
|
result.repointed.push(e.path);
|
|
653
|
-
} catch {
|
|
655
|
+
} catch {
|
|
656
|
+
// The old link is already gone at this point (unlinkSync above
|
|
657
|
+
// succeeded). Mirror installSkill()'s fallback: try a copy rather than
|
|
658
|
+
// leaving the skill MISSING on disk while the manifest still claims
|
|
659
|
+
// it's installed (e.g. a Windows cross-volume junction failure).
|
|
660
|
+
// Stage into a dot-prefixed temp dir and rename into place, same as
|
|
661
|
+
// the copy-refresh path above — copying directly into e.path left a
|
|
662
|
+
// partial, SKILL.md-less directory permanently orphaned on disk
|
|
663
|
+
// (unrecognized by isOwnedCopy) when copyDir failed partway (disk
|
|
664
|
+
// full, interrupted process).
|
|
665
|
+
const dir = path.dirname(e.path);
|
|
666
|
+
const base = path.basename(e.path);
|
|
667
|
+
const suffix = crypto.randomBytes(4).toString('hex');
|
|
668
|
+
const fresh = path.join(dir, `.${base}.new-${suffix}`);
|
|
669
|
+
try {
|
|
670
|
+
copyDir(source, fresh);
|
|
671
|
+
fs.renameSync(fresh, e.path);
|
|
672
|
+
keep.push({ ...e, type: 'copy', source });
|
|
673
|
+
result.repointed.push(e.path);
|
|
674
|
+
} catch {
|
|
675
|
+
try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
|
|
676
|
+
// Both the re-point and the copy fallback failed — the skill really
|
|
677
|
+
// is gone from disk now. Drop the entry instead of leaving the
|
|
678
|
+
// manifest claiming a missing skill is still installed.
|
|
679
|
+
result.pruned.push(e.path);
|
|
680
|
+
}
|
|
681
|
+
}
|
|
654
682
|
} else if (!targetLive) {
|
|
655
683
|
// Dangling with no resolvable source (package fully removed): remove the
|
|
656
684
|
// dead link and drop the entry so the agent's skills dir stays clean.
|
|
@@ -24,6 +24,7 @@ import { ServiceNames } from '../core/service-interfaces.ts';
|
|
|
24
24
|
import type { IResearchOrchestration, IResearchSynthesisService } from '../core/service-interfaces.ts';
|
|
25
25
|
import { metrics, MetricsRegistry, runWithRunRegistry } from '../utils/metrics.ts';
|
|
26
26
|
import { createResearchRunId, logger, createLogger, isVerboseFromEnv, runWithLogger } from '../logger.ts';
|
|
27
|
+
import { redactSecrets } from '../utils/log-utils.ts';
|
|
27
28
|
import { exportResearchReport, appendExportMessage } from '../utils/research-export.ts';
|
|
28
29
|
import { validateAndSanitizeQuery } from '../utils/input-validation.ts';
|
|
29
30
|
import { validateInitialLinks, MAX_INITIAL_LINKS, MAX_INITIAL_LINK_CHARS } from '../utils/url-utils.ts';
|
|
@@ -507,14 +508,14 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
|
|
|
507
508
|
return {
|
|
508
509
|
content: [{
|
|
509
510
|
type: 'text',
|
|
510
|
-
text: `[SYSTEM MESSAGE]: The research operation was halted gracefully because an API rate limit (HTTP 429) was reached. Please inform the user that the operation was stopped due to provider rate limits and they should wait a moment before trying again.\n\nDetails: ${String(error)}`
|
|
511
|
+
text: `[SYSTEM MESSAGE]: The research operation was halted gracefully because an API rate limit (HTTP 429) was reached. Please inform the user that the operation was stopped due to provider rate limits and they should wait a moment before trying again.\n\nDetails: ${redactSecrets(String(error))}`
|
|
511
512
|
}],
|
|
512
513
|
details: {}
|
|
513
514
|
};
|
|
514
515
|
}
|
|
515
516
|
|
|
516
517
|
logger.error('[research] run failed', error);
|
|
517
|
-
return { content: [{ type: 'text', text: `Research failed: ${String(error)}` }], details: {} };
|
|
518
|
+
return { content: [{ type: 'text', text: `Research failed: ${redactSecrets(String(error))}` }], details: {} };
|
|
518
519
|
} finally {
|
|
519
520
|
// Guarantees panel teardown even when the run fails before the inner
|
|
520
521
|
// research loop is reached (e.g. health-check failure), where the inner
|
package/src/tools/scrape.ts
CHANGED
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
MAX_SCRAPE_CONTENT_CHARS_PER_DOC,
|
|
25
25
|
} from '../constants.ts';
|
|
26
26
|
import { truncateWithMarker } from '../utils/text-utils.ts';
|
|
27
|
+
import { redactSecrets } from '../utils/log-utils.ts';
|
|
27
28
|
import { type Config, getConfig } from '../config.ts';
|
|
28
29
|
import { getService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
|
|
29
30
|
import { ServiceNames } from '../core/service-interfaces.ts';
|
|
@@ -261,16 +262,17 @@ export function createScrapeTool(options: {
|
|
|
261
262
|
const concurrency = p.maxConcurrency || defaultConcurrency;
|
|
262
263
|
|
|
263
264
|
try {
|
|
264
|
-
const results = await scrape(urlsToFetch, concurrency, signal,
|
|
265
|
+
const results = await scrape(urlsToFetch, concurrency, signal, config, getGlobalState().researchId, (result) => {
|
|
265
266
|
options.onUrlScrapeResult?.(result.url, result.success);
|
|
266
267
|
}, container);
|
|
267
268
|
freshResults = Array.isArray(results) ? results : [];
|
|
268
269
|
} catch (error) {
|
|
269
|
-
|
|
270
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
271
|
+
logger.error(`[scrape tool] Scrape failed: ${message}`);
|
|
270
272
|
metrics.increment('tool_scrape_calls_total', 1, { status: 'error' });
|
|
271
273
|
return {
|
|
272
|
-
content: [{ type: 'text', text: `# Scrape Failed\n\n${
|
|
273
|
-
details: { error: String(error) },
|
|
274
|
+
content: [{ type: 'text', text: `# Scrape Failed\n\n${redactSecrets(message)}` }],
|
|
275
|
+
details: { error: redactSecrets(String(error)) },
|
|
274
276
|
};
|
|
275
277
|
}
|
|
276
278
|
|
package/src/tools/security.ts
CHANGED
|
@@ -13,6 +13,15 @@ import { getMaxGatheringCalls } from '../constants.ts';
|
|
|
13
13
|
import { metrics } from '../utils/metrics.ts';
|
|
14
14
|
import type { SecuritySearchParams } from '../security/types.ts';
|
|
15
15
|
|
|
16
|
+
const MAX_DESCRIPTION_CHARS = 300;
|
|
17
|
+
|
|
18
|
+
/** Truncates an advisory description for the markdown summary line. */
|
|
19
|
+
function truncateDescription(description: string): string {
|
|
20
|
+
return description.length > MAX_DESCRIPTION_CHARS
|
|
21
|
+
? `${description.substring(0, MAX_DESCRIPTION_CHARS)}...`
|
|
22
|
+
: description;
|
|
23
|
+
}
|
|
24
|
+
|
|
16
25
|
export function createSecuritySearchTool(options: {
|
|
17
26
|
ctx: ExtensionContext;
|
|
18
27
|
tracker: ToolUsageTracker;
|
|
@@ -229,7 +238,7 @@ export function createSecuritySearchTool(options: {
|
|
|
229
238
|
}
|
|
230
239
|
}
|
|
231
240
|
const description = vuln.description;
|
|
232
|
-
markdown += `- **Description:** ${description
|
|
241
|
+
markdown += `- **Description:** ${truncateDescription(description)}\n`;
|
|
233
242
|
if (vuln.knownExploited === true) {
|
|
234
243
|
markdown += '- **[Actively Exploited]**\n';
|
|
235
244
|
}
|
|
@@ -270,7 +279,7 @@ export function createSecuritySearchTool(options: {
|
|
|
270
279
|
markdown += `- **Product:** ${vuln.product}\n`;
|
|
271
280
|
}
|
|
272
281
|
const description = vuln.description;
|
|
273
|
-
markdown += `- **Description:** ${description
|
|
282
|
+
markdown += `- **Description:** ${truncateDescription(description)}\n`;
|
|
274
283
|
if (vuln.dueDate !== undefined) {
|
|
275
284
|
markdown += `- **Due Date:** ${vuln.dueDate}\n`;
|
|
276
285
|
}
|
|
@@ -310,7 +319,7 @@ export function createSecuritySearchTool(options: {
|
|
|
310
319
|
}
|
|
311
320
|
markdown += `- **Summary:** ${adv.summary}\n`;
|
|
312
321
|
const description = adv.description ?? '';
|
|
313
|
-
markdown += `- **Description:** ${description
|
|
322
|
+
markdown += `- **Description:** ${truncateDescription(description)}\n`;
|
|
314
323
|
if (adv.published) {
|
|
315
324
|
markdown += `- **Published:** ${adv.published}\n`;
|
|
316
325
|
}
|
|
@@ -344,7 +353,7 @@ export function createSecuritySearchTool(options: {
|
|
|
344
353
|
markdown += `### ${vuln.id}\n`;
|
|
345
354
|
markdown += `- **Severity:** ${vuln.severity}\n`;
|
|
346
355
|
const description = vuln.description;
|
|
347
|
-
markdown += `- **Description:** ${description
|
|
356
|
+
markdown += `- **Description:** ${truncateDescription(description)}\n`;
|
|
348
357
|
if (vuln.affectedProducts && vuln.affectedProducts.length > 0) {
|
|
349
358
|
markdown += `- **Affected:** ${vuln.affectedProducts.join(', ')}\n`;
|
|
350
359
|
}
|
|
@@ -10,6 +10,25 @@ export interface RotationLogger {
|
|
|
10
10
|
log(...args: unknown[]): void;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
+
// Numeric equivalent of the 'a' flag string (O_WRONLY|O_APPEND|O_CREAT), with
|
|
14
|
+
// O_NOFOLLOW added when the platform defines it (not available on Windows) so
|
|
15
|
+
// the open refuses to follow a symlink an attacker with write access to the
|
|
16
|
+
// log directory could plant at this path in the window between the preceding
|
|
17
|
+
// unlink/rename and this reopen — defense-in-depth on top of an already-narrow
|
|
18
|
+
// threat model (it requires local write access to this directory to begin with).
|
|
19
|
+
const APPEND_CREATE_NOFOLLOW_FLAGS =
|
|
20
|
+
fs.constants.O_WRONLY | fs.constants.O_APPEND | fs.constants.O_CREAT |
|
|
21
|
+
(typeof fs.constants.O_NOFOLLOW === 'number' ? fs.constants.O_NOFOLLOW : 0);
|
|
22
|
+
|
|
23
|
+
/** Best-effort (re)create `filePath` with `mode`, refusing to follow a symlink
|
|
24
|
+
* planted at the path. Failures are swallowed by design — see call sites.
|
|
25
|
+
* Exported for unit testing of the symlink-refusal behavior. */
|
|
26
|
+
export function touchOwnerOnly(filePath: string, mode: number): void {
|
|
27
|
+
try {
|
|
28
|
+
fs.closeSync(fs.openSync(filePath, APPEND_CREATE_NOFOLLOW_FLAGS, mode));
|
|
29
|
+
} catch { /* best-effort — the next append recreates the file */ }
|
|
30
|
+
}
|
|
31
|
+
|
|
13
32
|
export class LogRotation {
|
|
14
33
|
private readonly MAX_LOG_SIZE = 10 * 1024 * 1024; // 10MB max file size
|
|
15
34
|
private readonly MAX_LOG_FILES = 10; // Keep last 10 archived logs
|
|
@@ -30,9 +49,7 @@ export class LogRotation {
|
|
|
30
49
|
// Same rationale as the post-rotation recreate in rotateLogFile: the
|
|
31
50
|
// unlink discarded the 0o600 file, and a umask-default recreate by the
|
|
32
51
|
// next append would drop the owner-only posture.
|
|
33
|
-
|
|
34
|
-
fs.closeSync(fs.openSync(logFile, 'a', 0o600));
|
|
35
|
-
} catch { /* best-effort — the next append recreates the file */ }
|
|
52
|
+
touchOwnerOnly(logFile, 0o600);
|
|
36
53
|
// And, as there, re-stamp the mode: a concurrent append landing between
|
|
37
54
|
// the unlink and the open makes the creation-time 0o600 a no-op.
|
|
38
55
|
try {
|
|
@@ -94,9 +111,7 @@ export class LogRotation {
|
|
|
94
111
|
// world-traversable tmpdir, silently dropping the owner-only posture Logger
|
|
95
112
|
// establishes at construction. Mode applies only at creation, so this is
|
|
96
113
|
// harmless where a file already exists or mode semantics differ (win32).
|
|
97
|
-
|
|
98
|
-
fs.closeSync(fs.openSync(logFile, 'a', 0o600));
|
|
99
|
-
} catch { /* best-effort — the next append recreates the file */ }
|
|
114
|
+
touchOwnerOnly(logFile, 0o600);
|
|
100
115
|
// The mode above applies only if THIS open created the file. Several
|
|
101
116
|
// processes append to this consolidated log, so a concurrent async append
|
|
102
117
|
// can recreate it at umask perms between the rename and the open, turning
|
package/src/utils/log-utils.ts
CHANGED
|
@@ -133,6 +133,16 @@ const CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
|
|
|
133
133
|
* Applied to every message before it reaches the log file or the console, so
|
|
134
134
|
* neither sink receives clear-text secrets or unbounded network data.
|
|
135
135
|
*/
|
|
136
|
+
// How much raw input the regex chain below is allowed to scan before a final
|
|
137
|
+
// truncation to MAX_LOG_MESSAGE_LENGTH. A multiple of MAX_LOG_MESSAGE_LENGTH,
|
|
138
|
+
// not 1:1, because most patterns below SHRINK text (a 64-char secret becomes
|
|
139
|
+
// "[REDACTED]"), so more than `keep` raw characters can be needed to fully
|
|
140
|
+
// populate the final truncated output. Nothing past this point can ever
|
|
141
|
+
// survive into the visible output regardless, so skipping it is free — this
|
|
142
|
+
// is what bounds redactSecrets() to O(MAX_LOG_MESSAGE_LENGTH) instead of
|
|
143
|
+
// O(input length) on multi-megabyte scraped payloads.
|
|
144
|
+
const REDACT_SCAN_LENGTH = MAX_LOG_MESSAGE_LENGTH * 4;
|
|
145
|
+
|
|
136
146
|
export function redactSecrets(message: string): string {
|
|
137
147
|
// Redact BEFORE truncating. Every pattern below needs to see a secret in
|
|
138
148
|
// full to match it (e.g. LONG_HEX_SECRET_PATTERN requires 32+ contiguous
|
|
@@ -140,7 +150,8 @@ export function redactSecrets(message: string): string {
|
|
|
140
150
|
// whatever fragment survives the cut (e.g. the first 20 chars of a 64-hex
|
|
141
151
|
// browser auth secret) too short for any pattern to recognize, so it was
|
|
142
152
|
// written out in the clear right before the "…[truncated N chars]" marker.
|
|
143
|
-
|
|
153
|
+
const rawLength = message.length;
|
|
154
|
+
let out = rawLength > REDACT_SCAN_LENGTH ? message.slice(0, REDACT_SCAN_LENGTH) : message;
|
|
144
155
|
out = out.replace(ANSI_PATTERN, '');
|
|
145
156
|
out = out.replace(URL_CREDENTIALS_PATTERN, '$1[REDACTED]@');
|
|
146
157
|
// Redact whole-token formats BEFORE the key/value pass: the KV value class
|
|
@@ -172,7 +183,7 @@ export function redactSecrets(message: string): string {
|
|
|
172
183
|
if (lastKeptUnit >= 0xd800 && lastKeptUnit <= 0xdbff) keep -= 1;
|
|
173
184
|
}
|
|
174
185
|
if (out.length > MAX_LOG_MESSAGE_LENGTH) {
|
|
175
|
-
out = `${out.slice(0, keep)}…[truncated ${
|
|
186
|
+
out = `${out.slice(0, keep)}…[truncated ${rawLength - keep} chars]`;
|
|
176
187
|
}
|
|
177
188
|
return out;
|
|
178
189
|
}
|
package/src/utils/text-utils.ts
CHANGED
|
@@ -188,8 +188,12 @@ export function formatDuration(ms: number): string {
|
|
|
188
188
|
if (ms < 1000) return `${ms}ms`;
|
|
189
189
|
const seconds = ms / 1000;
|
|
190
190
|
if (seconds < 60) return `${seconds.toFixed(1)}s`;
|
|
191
|
-
|
|
192
|
-
|
|
191
|
+
// Round the total first, then derive minutes/seconds from that integer —
|
|
192
|
+
// rounding remainingSeconds independently of minutes let e.g. 59.6s round up
|
|
193
|
+
// to a nonsensical "Xm 60s" instead of rolling over to the next minute.
|
|
194
|
+
const totalSeconds = Math.round(seconds);
|
|
195
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
196
|
+
const remainingSeconds = totalSeconds % 60;
|
|
193
197
|
if (minutes < 60) return `${minutes}m ${remainingSeconds}s`;
|
|
194
198
|
const hours = Math.floor(minutes / 60);
|
|
195
199
|
const remainingMinutes = minutes % 60;
|
package/src/youtube/potoken.ts
CHANGED
|
@@ -42,6 +42,7 @@
|
|
|
42
42
|
*/
|
|
43
43
|
|
|
44
44
|
import { logger } from '../logger.ts';
|
|
45
|
+
import { safeUnref } from '../utils/safe-unref.ts';
|
|
45
46
|
import { metrics } from '../utils/metrics.ts';
|
|
46
47
|
|
|
47
48
|
/**
|
|
@@ -76,6 +77,15 @@ function describeError(error: unknown): string {
|
|
|
76
77
|
// Module-level mutex: serialize all globalThis mutation across concurrent callers.
|
|
77
78
|
let mintChain: Promise<unknown> = Promise.resolve();
|
|
78
79
|
|
|
80
|
+
// How long the mutex waits for a straggling doMint() before force-releasing
|
|
81
|
+
// the chain for the next caller. doMint() awaits non-fetch operations with no
|
|
82
|
+
// abort signal (interpreter eval, BotGuard VM execution against a
|
|
83
|
+
// hostile/malformed attestation script) that could in principle hang forever;
|
|
84
|
+
// without this bound, a single wedged call would permanently break YouTube
|
|
85
|
+
// transcript fetching for the rest of the process. Mirrors the hard-release
|
|
86
|
+
// timer on the browser page-creation mutex in thread-worker-messaging.ts.
|
|
87
|
+
const MINT_HARD_RELEASE_MS = 5 * 60 * 1000;
|
|
88
|
+
|
|
79
89
|
/**
|
|
80
90
|
* Mint a session-bound token (for `identifiers[0]`, the visitorData) plus one
|
|
81
91
|
* content-bound token per remaining identifier (the video IDs), all from a
|
|
@@ -90,8 +100,22 @@ export async function mintPoTokens(
|
|
|
90
100
|
): Promise<Map<string, string>> {
|
|
91
101
|
// Chain on the mutex so only one bridged section runs at a time.
|
|
92
102
|
const run = mintChain.then(() => doMint(identifiers, opts), () => doMint(identifiers, opts));
|
|
93
|
-
// Keep the chain alive regardless of this call's success/failure
|
|
94
|
-
|
|
103
|
+
// Keep the chain alive regardless of this call's success/failure, but bound
|
|
104
|
+
// how long the NEXT caller waits on it: if `run` never settles, force the
|
|
105
|
+
// chain open after the hard timeout rather than wedging every future
|
|
106
|
+
// mintPoTokens() call behind it forever. The abandoned `run` may still
|
|
107
|
+
// mutate globalThis in the background after this — an accepted trade-off
|
|
108
|
+
// in the pathological case, same as the sibling browser-worker fix.
|
|
109
|
+
const settled = run.then(() => undefined, () => undefined);
|
|
110
|
+
let hardTimer: NodeJS.Timeout;
|
|
111
|
+
const hardRelease = new Promise<void>(resolve => {
|
|
112
|
+
hardTimer = setTimeout(() => {
|
|
113
|
+
logger.warn('[youtube] potoken mint mutex force-released — doMint() did not settle within the hard timeout');
|
|
114
|
+
resolve();
|
|
115
|
+
}, MINT_HARD_RELEASE_MS);
|
|
116
|
+
safeUnref(hardTimer);
|
|
117
|
+
});
|
|
118
|
+
mintChain = Promise.race([settled, hardRelease]).finally(() => clearTimeout(hardTimer));
|
|
95
119
|
return run;
|
|
96
120
|
}
|
|
97
121
|
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
import { withTimeout, retryWithBackoff, createTimeoutSignal } from '../web-research/retry-utils.ts';
|
|
21
21
|
import { CircuitBreaker } from '../utils/circuit-breaker.ts';
|
|
22
22
|
import { logger } from '../logger.ts';
|
|
23
|
+
import { redactSecrets } from '../utils/log-utils.ts';
|
|
23
24
|
import { metrics } from '../utils/metrics.ts';
|
|
24
25
|
import { readTextCapped } from '../utils/http-body.ts';
|
|
25
26
|
import { mintPoTokens } from './potoken.ts';
|
|
@@ -355,8 +356,10 @@ function parseJson3(body: string): string {
|
|
|
355
356
|
// Log the real cause: an unparseable body is NOT necessarily an empty
|
|
356
357
|
// bot-protection response (the caller's default diagnosis) — it can be an HTML
|
|
357
358
|
// interstitial or a response-shape change. A short snippet disambiguates without
|
|
358
|
-
// dumping a huge/again-sensitive body.
|
|
359
|
-
|
|
359
|
+
// dumping a huge/again-sensitive body. Redact BEFORE truncating to 120 chars —
|
|
360
|
+
// slicing first can cut a secret-shaped token mid-way, leaving a fragment too
|
|
361
|
+
// short for redactSecrets' whole-token patterns to recognize.
|
|
362
|
+
logger.debug(`[youtube] timedtext body was not valid json3 (first 120 chars): ${redactSecrets(body).slice(0, 120)}`);
|
|
360
363
|
return '';
|
|
361
364
|
}
|
|
362
365
|
const events = (parsed as { events?: unknown[] }).events;
|