@lincoln504/pi-research 1.3.3 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.mjs +177 -72
- package/dist/thread-worker.mjs +56 -37
- package/docs/CHANGELOG.md +39 -0
- package/package.json +1 -1
- package/skills/pi-research/SKILL.md +1 -1
- package/src/core/service-registry.ts +37 -3
- package/src/healthcheck/index.ts +9 -7
- package/src/infrastructure/browser/task-execution-service.ts +21 -5
- package/src/infrastructure/browser/thread-worker-browser.ts +20 -1
- package/src/infrastructure/browser/thread-worker-messaging.ts +30 -22
- package/src/infrastructure/browser/thread-worker.mjs +56 -37
- package/src/infrastructure/knowledge-store-service.ts +27 -2
- package/src/knowledge/chunker.ts +14 -1
- package/src/orchestration/quick-research-orchestrator.ts +13 -2
- package/src/orchestration/research-orchestration-service.ts +16 -3
- package/src/sdk.ts +11 -2
- package/src/security/github-advisories.ts +13 -4
- package/src/skill-install/skill-installer.ts +40 -5
- package/src/stackexchange/index.ts +15 -0
- package/src/stackexchange/rest-client.ts +13 -24
- package/src/utils/input-validation.ts +0 -10
- package/src/utils/json-utils.ts +0 -78
- package/src/utils/log-utils.ts +32 -33
- package/src/utils/metrics.ts +0 -8
- package/src/utils/shared-links.ts +0 -14
- package/src/utils/stdio-capture.ts +44 -7
- package/src/utils/url-utils.ts +7 -1
- package/src/web-research/scraper-utils.ts +8 -3
- package/src/web-research/web-scraper.ts +33 -1
package/README.md
CHANGED
|
@@ -61,7 +61,7 @@ ask in natural language and the tool picks the right one:
|
|
|
61
61
|
### Requirements / limitations
|
|
62
62
|
|
|
63
63
|
- Node.js >= 22.19.0
|
|
64
|
-
- An LLM with a 100k+ context window (bring your own key)
|
|
64
|
+
- An LLM with a 100k+ context window (bring your own key or use a local model)
|
|
65
65
|
- Internet access
|
|
66
66
|
- A residential IP address — search, scraping, and YouTube transcripts all rely on a residential connection. A datacenter/VPS/cloud IP gets bot-blocked by the providers these features depend on.
|
|
67
67
|
- The pi runtime the engine builds on — `@earendil-works/pi-ai`, `@earendil-works/pi-coding-agent`, and `@earendil-works/pi-tui`. The pi extension uses the host's copies; the standalone CLI / agent skill install them as dependencies.
|
package/dist/cli.mjs
CHANGED
|
@@ -215,12 +215,7 @@ function formatArg(arg) {
|
|
|
215
215
|
return String(arg);
|
|
216
216
|
}
|
|
217
217
|
function redactSecrets(message) {
|
|
218
|
-
let
|
|
219
|
-
if (message.length > keep) {
|
|
220
|
-
const lastKeptUnit = message.charCodeAt(keep - 1);
|
|
221
|
-
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
222
|
-
}
|
|
223
|
-
let out = message.length > MAX_LOG_MESSAGE_LENGTH ? `${message.slice(0, keep)}\u2026[truncated ${message.length - keep} chars]` : message;
|
|
218
|
+
let out = message;
|
|
224
219
|
out = out.replace(ANSI_PATTERN, "");
|
|
225
220
|
out = out.replace(URL_CREDENTIALS_PATTERN, "$1[REDACTED]@");
|
|
226
221
|
out = out.replace(JWT_PATTERN, "[REDACTED]");
|
|
@@ -231,6 +226,14 @@ function redactSecrets(message) {
|
|
|
231
226
|
out = out.replace(KNOWN_TOKEN_PATTERN, "[REDACTED]");
|
|
232
227
|
out = out.replace(PROVIDER_TOKEN_PATTERN, "[REDACTED]");
|
|
233
228
|
out = out.replace(LONG_HEX_SECRET_PATTERN, "[REDACTED]");
|
|
229
|
+
let keep = MAX_LOG_MESSAGE_LENGTH;
|
|
230
|
+
if (out.length > keep) {
|
|
231
|
+
const lastKeptUnit = out.charCodeAt(keep - 1);
|
|
232
|
+
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
233
|
+
}
|
|
234
|
+
if (out.length > MAX_LOG_MESSAGE_LENGTH) {
|
|
235
|
+
out = `${out.slice(0, keep)}\u2026[truncated ${out.length - keep} chars]`;
|
|
236
|
+
}
|
|
234
237
|
return out;
|
|
235
238
|
}
|
|
236
239
|
function stripTerminalEscapes(message) {
|
|
@@ -245,7 +248,7 @@ var init_log_utils = __esm({
|
|
|
245
248
|
ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
|
|
246
249
|
URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
|
|
247
250
|
COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
|
|
248
|
-
SENSITIVE_KV_PATTERN = /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
|
|
251
|
+
SENSITIVE_KV_PATTERN = /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
|
|
249
252
|
KNOWN_TOKEN_PATTERN = /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
|
|
250
253
|
JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
|
|
251
254
|
BASIC_AUTH_PATTERN = /\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/gi;
|
|
@@ -497,7 +500,10 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
497
500
|
unkeyedCaptureDepth++;
|
|
498
501
|
isAnyLoggerCapturingOutput = true;
|
|
499
502
|
}
|
|
500
|
-
const
|
|
503
|
+
const stderrDecoder = new TextDecoder2();
|
|
504
|
+
const stdoutDecoder = new TextDecoder2();
|
|
505
|
+
const fd1SyncDecoder = new TextDecoder2();
|
|
506
|
+
const fd2SyncDecoder = new TextDecoder2();
|
|
501
507
|
const originalStderrWrite = process.stderr.write;
|
|
502
508
|
const originalStdoutWrite = process.stdout.write;
|
|
503
509
|
const originalFsWriteSync = fs3.writeSync;
|
|
@@ -543,7 +549,11 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
543
549
|
console.debug = createConsolePatch("debug", logFile, hasSufficientDiskSpace);
|
|
544
550
|
process.stderr.write = (chunk, encodingOrCb, callback) => {
|
|
545
551
|
const cb = typeof encodingOrCb === "function" ? encodingOrCb : callback;
|
|
546
|
-
const message = typeof chunk === "string" ? chunk :
|
|
552
|
+
const message = typeof chunk === "string" ? chunk : stderrDecoder.decode(chunk, { stream: true });
|
|
553
|
+
if (message.length === 0) {
|
|
554
|
+
if (typeof cb === "function") cb();
|
|
555
|
+
return true;
|
|
556
|
+
}
|
|
547
557
|
if (!hasSufficientDiskSpace()) {
|
|
548
558
|
if (typeof cb === "function") cb();
|
|
549
559
|
return true;
|
|
@@ -564,7 +574,7 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
564
574
|
return true;
|
|
565
575
|
};
|
|
566
576
|
process.stdout.write = (chunk, encodingOrCb, callback) => {
|
|
567
|
-
const message = typeof chunk === "string" ? chunk :
|
|
577
|
+
const message = typeof chunk === "string" ? chunk : stdoutDecoder.decode(chunk, { stream: true });
|
|
568
578
|
const isNative = isNativeLog(message);
|
|
569
579
|
const complexTui = isComplexTui(message);
|
|
570
580
|
const boxDrawing = isBoxDrawing(message);
|
|
@@ -618,7 +628,11 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
618
628
|
if (!descriptor || (descriptor.writable || descriptor.set)) {
|
|
619
629
|
fs3.writeSync = (fd, chunk, ...args) => {
|
|
620
630
|
if (fd === 1 || fd === 2) {
|
|
621
|
-
const
|
|
631
|
+
const fdDecoder = fd === 1 ? fd1SyncDecoder : fd2SyncDecoder;
|
|
632
|
+
const message = typeof chunk === "string" ? chunk : fdDecoder.decode(chunk, { stream: true });
|
|
633
|
+
if (fd === 2 && message.length === 0) {
|
|
634
|
+
return typeof chunk === "string" ? Buffer.from(chunk).length : chunk.length;
|
|
635
|
+
}
|
|
622
636
|
const isNative = isNativeLog(message);
|
|
623
637
|
const boxDrawing = isBoxDrawing(message);
|
|
624
638
|
const complexTui = isComplexTui(message);
|
|
@@ -1349,6 +1363,24 @@ var init_service_registry = __esm({
|
|
|
1349
1363
|
]);
|
|
1350
1364
|
if (settleTimer) clearTimeout(settleTimer);
|
|
1351
1365
|
}
|
|
1366
|
+
const inFlightDisposals = Array.from(this.services.values()).map((r) => r.disposalPromise).filter((p) => p != null);
|
|
1367
|
+
if (inFlightDisposals.length > 0) {
|
|
1368
|
+
if (this.defaultOptions.enableLogging) {
|
|
1369
|
+
logger.debug(`[ServiceContainer] Waiting for ${inFlightDisposals.length} in-flight disposal(s) before teardown`);
|
|
1370
|
+
}
|
|
1371
|
+
let disposalSettleTimer;
|
|
1372
|
+
await Promise.race([
|
|
1373
|
+
Promise.allSettled(inFlightDisposals),
|
|
1374
|
+
new Promise((resolve9) => {
|
|
1375
|
+
disposalSettleTimer = setTimeout(() => {
|
|
1376
|
+
logger.warn("[ServiceContainer] Timed out waiting for in-flight service disposal; proceeding with teardown");
|
|
1377
|
+
resolve9();
|
|
1378
|
+
}, DISPOSE_INFLIGHT_INIT_TIMEOUT_MS);
|
|
1379
|
+
disposalSettleTimer.unref?.();
|
|
1380
|
+
})
|
|
1381
|
+
]);
|
|
1382
|
+
if (disposalSettleTimer) clearTimeout(disposalSettleTimer);
|
|
1383
|
+
}
|
|
1352
1384
|
const activeServices = Array.from(this.services.keys()).filter(
|
|
1353
1385
|
(name) => this.services.get(name)?.instance !== null
|
|
1354
1386
|
);
|
|
@@ -1932,7 +1964,7 @@ function normalizeUrl(url) {
|
|
|
1932
1964
|
}
|
|
1933
1965
|
return result;
|
|
1934
1966
|
} catch (_err) {
|
|
1935
|
-
logger.debug(`[url-utils] normalizing unparseable URL (${url.slice(0, 200)}): ${_err instanceof Error ? _err.message : String(_err)}`);
|
|
1967
|
+
logger.debug(`[url-utils] normalizing unparseable URL (${redactSecrets(url).slice(0, 200)}): ${_err instanceof Error ? _err.message : String(_err)}`);
|
|
1936
1968
|
let cleaned = url.trim().replace(/^[*_~`"']{1,20}/, "").replace(/[*_~`"',.)}\]]{1,20}$/, "");
|
|
1937
1969
|
const hashIdx = cleaned.indexOf("#");
|
|
1938
1970
|
if (hashIdx >= 0) cleaned = cleaned.substring(0, hashIdx);
|
|
@@ -1996,6 +2028,7 @@ var init_url_utils = __esm({
|
|
|
1996
2028
|
"src/utils/url-utils.ts"() {
|
|
1997
2029
|
"use strict";
|
|
1998
2030
|
init_logger();
|
|
2031
|
+
init_log_utils();
|
|
1999
2032
|
MAX_INITIAL_LINKS = 20;
|
|
2000
2033
|
MAX_INITIAL_LINK_CHARS = 2048;
|
|
2001
2034
|
}
|
|
@@ -6434,7 +6467,7 @@ var init_chunker = __esm({
|
|
|
6434
6467
|
if (code >= 56320 && code <= 57343) {
|
|
6435
6468
|
if (start - 1 > iterationStart) {
|
|
6436
6469
|
start -= 1;
|
|
6437
|
-
} else if (start + 1
|
|
6470
|
+
} else if (start + 1 <= end) {
|
|
6438
6471
|
start += 1;
|
|
6439
6472
|
}
|
|
6440
6473
|
}
|
|
@@ -8663,6 +8696,9 @@ var init_knowledge_store_service = __esm({
|
|
|
8663
8696
|
await this._initializationPromise;
|
|
8664
8697
|
} catch {
|
|
8665
8698
|
}
|
|
8699
|
+
if (this._disposalPromise) {
|
|
8700
|
+
return this._disposalPromise;
|
|
8701
|
+
}
|
|
8666
8702
|
}
|
|
8667
8703
|
this.lifecycle = "disposing" /* DISPOSING */;
|
|
8668
8704
|
logger.debug("[KnowledgeStoreService] Disposing...");
|
|
@@ -8670,7 +8706,8 @@ var init_knowledge_store_service = __esm({
|
|
|
8670
8706
|
const store = this._store;
|
|
8671
8707
|
const writerQueue = this._writerQueue;
|
|
8672
8708
|
const initLock = this._initLock;
|
|
8673
|
-
|
|
8709
|
+
let disposePromise = null;
|
|
8710
|
+
disposePromise = (async () => {
|
|
8674
8711
|
try {
|
|
8675
8712
|
if (writerQueue) {
|
|
8676
8713
|
await writerQueue.dispose?.();
|
|
@@ -8694,9 +8731,12 @@ var init_knowledge_store_service = __esm({
|
|
|
8694
8731
|
logger.error("[KnowledgeStoreService] Error during disposal:", err);
|
|
8695
8732
|
} finally {
|
|
8696
8733
|
this.lifecycle = "disposed" /* DISPOSED */;
|
|
8697
|
-
this._disposalPromise
|
|
8734
|
+
if (this._disposalPromise === disposePromise) {
|
|
8735
|
+
this._disposalPromise = null;
|
|
8736
|
+
}
|
|
8698
8737
|
}
|
|
8699
8738
|
})();
|
|
8739
|
+
this._disposalPromise = disposePromise;
|
|
8700
8740
|
return this._disposalPromise;
|
|
8701
8741
|
}
|
|
8702
8742
|
/**
|
|
@@ -9740,16 +9780,31 @@ function installSkill(toolIds, opts = {}) {
|
|
|
9740
9780
|
continue;
|
|
9741
9781
|
}
|
|
9742
9782
|
fs16.unlinkSync(absSkillPath);
|
|
9743
|
-
|
|
9744
|
-
|
|
9745
|
-
|
|
9783
|
+
try {
|
|
9784
|
+
fs16.symlinkSync(source, absSkillPath, process.platform === "win32" ? "junction" : "dir");
|
|
9785
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: "symlink", source, createdAt: now, ...skillVersion ? { version: skillVersion } : {} });
|
|
9786
|
+
results.push({ tool: id, path: absSkillPath, type: "symlink", status: "installed", message: "re-pointed stale symlink to current source" });
|
|
9787
|
+
} catch (relinkErr) {
|
|
9788
|
+
try {
|
|
9789
|
+
copyDir(source, absSkillPath);
|
|
9790
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: "copy", source, createdAt: now, ...skillVersion ? { version: skillVersion } : {} });
|
|
9791
|
+
results.push({ tool: id, path: absSkillPath, type: "copy", status: "installed", message: `symlink re-point unavailable (${relinkErr?.code ?? relinkErr?.message}); copied instead` });
|
|
9792
|
+
} catch (copyErr) {
|
|
9793
|
+
results.push({ tool: id, path: absSkillPath, type: "symlink", status: "error", message: `symlink re-point failed (${relinkErr?.code ?? relinkErr?.message}) and the copy fallback also failed: ${copyErr?.message ?? String(copyErr)}` });
|
|
9794
|
+
}
|
|
9795
|
+
}
|
|
9746
9796
|
continue;
|
|
9747
9797
|
}
|
|
9748
9798
|
}
|
|
9749
9799
|
} catch {
|
|
9750
9800
|
}
|
|
9751
9801
|
results.push({ tool: id, path: absSkillPath, type, status: "already-installed" });
|
|
9752
|
-
|
|
9802
|
+
{
|
|
9803
|
+
const isSymlinkNow = isSymlinkPresent(absSkillPath);
|
|
9804
|
+
const existingEntry = manifest.entries.find((e) => e.path === absSkillPath);
|
|
9805
|
+
const versionField = isSymlinkNow ? skillVersion ? { version: skillVersion } : {} : existingEntry?.version ? { version: existingEntry.version } : {};
|
|
9806
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkNow ? "symlink" : "copy", source, createdAt: now, ...versionField });
|
|
9807
|
+
}
|
|
9753
9808
|
continue;
|
|
9754
9809
|
}
|
|
9755
9810
|
if (fs16.existsSync(absSkillPath) || isSymlinkPresent(absSkillPath)) {
|
|
@@ -15867,6 +15922,7 @@ init_logger();
|
|
|
15867
15922
|
|
|
15868
15923
|
// src/infrastructure/browser/task-execution-service.ts
|
|
15869
15924
|
init_logger();
|
|
15925
|
+
init_log_utils();
|
|
15870
15926
|
init_error_tracker();
|
|
15871
15927
|
|
|
15872
15928
|
// src/infrastructure/browser/browser-lifecycle.ts
|
|
@@ -15914,6 +15970,10 @@ var lastRestartTime = 0;
|
|
|
15914
15970
|
var RESTART_COOLDOWN_MS = 1e4;
|
|
15915
15971
|
var LEADER_HANDOVER_RETRIES = 3;
|
|
15916
15972
|
var HANDOVER_BACKOFF_MS = 1200;
|
|
15973
|
+
function briefErrorMessage(error) {
|
|
15974
|
+
const msg2 = error instanceof Error ? error.message : String(error);
|
|
15975
|
+
return redactSecrets(msg2).substring(0, 100);
|
|
15976
|
+
}
|
|
15917
15977
|
async function recoverFromLeaderHandover(container) {
|
|
15918
15978
|
logger.warn("[BrowserManager] Leader handover detected (pool draining) \u2014 dropping stale scheduler handle and re-resolving...");
|
|
15919
15979
|
await forceSchedulerRestart2(false, container).catch(
|
|
@@ -16003,7 +16063,7 @@ async function runBrowserTask(taskOrUrl, type = "scrape", config, signal, retrie
|
|
|
16003
16063
|
taskType: type,
|
|
16004
16064
|
errorType: "leader_handover"
|
|
16005
16065
|
});
|
|
16006
|
-
logger.warn(`[BrowserManager] Leader handover during ${type} task (handover attempts left: ${handoverRetries}): ${(error
|
|
16066
|
+
logger.warn(`[BrowserManager] Leader handover during ${type} task (handover attempts left: ${handoverRetries}): ${briefErrorMessage(error)}...`);
|
|
16007
16067
|
await recoverFromLeaderHandover(container);
|
|
16008
16068
|
await new Promise((resolve9) => setTimeout(resolve9, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
|
|
16009
16069
|
return runBrowserTask(taskOrUrl, type, config, signal, retries, container, handoverRetries - 1);
|
|
@@ -16015,7 +16075,7 @@ async function runBrowserTask(taskOrUrl, type = "scrape", config, signal, retrie
|
|
|
16015
16075
|
taskType: type,
|
|
16016
16076
|
errorType: "transient_socket_error"
|
|
16017
16077
|
});
|
|
16018
|
-
logger.warn(`[BrowserManager] Transient socket error during ${type} task (retries left: ${retries}): ${(error
|
|
16078
|
+
logger.warn(`[BrowserManager] Transient socket error during ${type} task (retries left: ${retries}): ${briefErrorMessage(error)}...`);
|
|
16019
16079
|
if (isPoolShutdownError(error)) {
|
|
16020
16080
|
logger.warn(`[BrowserManager] Pool is draining \u2014 waiting for pool idle before retry...`);
|
|
16021
16081
|
await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
|
|
@@ -16042,7 +16102,7 @@ async function runBrowserHealthCheck(config, retries = 1, signal, container = ge
|
|
|
16042
16102
|
operation: "healthcheck",
|
|
16043
16103
|
errorType: "transient_socket_error"
|
|
16044
16104
|
});
|
|
16045
|
-
logger.warn(`[BrowserManager] Transient socket error during healthcheck (retries left: ${retries}): ${(error
|
|
16105
|
+
logger.warn(`[BrowserManager] Transient socket error during healthcheck (retries left: ${retries}): ${briefErrorMessage(error)}...`);
|
|
16046
16106
|
if (isPoolShutdownError(error)) {
|
|
16047
16107
|
await recoverFromLeaderHandover(container);
|
|
16048
16108
|
} else {
|
|
@@ -16074,7 +16134,7 @@ async function runWorkerSearch(query, config, signal, retries = 1, sessionId, co
|
|
|
16074
16134
|
query,
|
|
16075
16135
|
errorType: "leader_handover"
|
|
16076
16136
|
});
|
|
16077
|
-
logger.warn(`[BrowserManager] Leader handover during search (handover attempts left: ${handoverRetries}): ${(error
|
|
16137
|
+
logger.warn(`[BrowserManager] Leader handover during search (handover attempts left: ${handoverRetries}): ${briefErrorMessage(error)}...`);
|
|
16078
16138
|
await recoverFromLeaderHandover(container);
|
|
16079
16139
|
await new Promise((resolve9) => setTimeout(resolve9, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
|
|
16080
16140
|
return runWorkerSearch(query, config, signal, retries, sessionId, container, handoverRetries - 1);
|
|
@@ -16086,7 +16146,7 @@ async function runWorkerSearch(query, config, signal, retries = 1, sessionId, co
|
|
|
16086
16146
|
query,
|
|
16087
16147
|
errorType: "transient_socket_error"
|
|
16088
16148
|
});
|
|
16089
|
-
logger.warn(`[BrowserManager] Transient socket error during search (retries left: ${retries}): ${(error
|
|
16149
|
+
logger.warn(`[BrowserManager] Transient socket error during search (retries left: ${retries}): ${briefErrorMessage(error)}...`);
|
|
16090
16150
|
if (isPoolShutdownError(error)) {
|
|
16091
16151
|
await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
|
|
16092
16152
|
} else {
|
|
@@ -16513,13 +16573,11 @@ function registerHealthChecks(registry, container = getServiceContainer()) {
|
|
|
16513
16573
|
registry.register("StateManager", async () => {
|
|
16514
16574
|
try {
|
|
16515
16575
|
const stateManager = await getService(ServiceNames.STATE_MANAGER, { container }, container);
|
|
16516
|
-
const stats = await stateManager.getMetrics();
|
|
16517
16576
|
const gpuOwner = await stateManager.getGpuOwner();
|
|
16518
16577
|
return {
|
|
16519
16578
|
healthy: true,
|
|
16520
16579
|
diagnostic: {
|
|
16521
16580
|
status: "operational",
|
|
16522
|
-
sessions: stats.activeSessions,
|
|
16523
16581
|
gpuLocked: !!gpuOwner,
|
|
16524
16582
|
gpuOwner: gpuOwner?.sessionId || "none"
|
|
16525
16583
|
}
|
|
@@ -16805,6 +16863,7 @@ function isBenignScrapeFailure(error) {
|
|
|
16805
16863
|
return msg2.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
|
|
16806
16864
|
msg2.startsWith("Fetch returned stub:") || // nav-only / near-empty page
|
|
16807
16865
|
msg2.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
|
|
16866
|
+
msg2.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
|
|
16808
16867
|
/^HTTP \d{3}\b/.test(msg2) || // remote 4xx/5xx status
|
|
16809
16868
|
isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
|
|
16810
16869
|
}
|
|
@@ -17097,8 +17156,22 @@ async function extractPdfToMarkdown(bytes) {
|
|
|
17097
17156
|
throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
|
|
17098
17157
|
}
|
|
17099
17158
|
const pdfExtractionStart = Date.now();
|
|
17159
|
+
let WasmPdfDocument;
|
|
17160
|
+
try {
|
|
17161
|
+
({ WasmPdfDocument } = await import("pdf-oxide-wasm"));
|
|
17162
|
+
} catch (e) {
|
|
17163
|
+
const msg2 = e instanceof Error ? e.message : String(e);
|
|
17164
|
+
logger.error(`[Scrapers] pdf-oxide-wasm failed to load \u2014 PDF extraction is unavailable (not just for this URL): ${msg2}`);
|
|
17165
|
+
metrics.increment("scrape_pdf_errors_total", 1, { error_type: "native_module_unavailable" });
|
|
17166
|
+
errorTracker.trackError(e instanceof Error ? e : String(e), {
|
|
17167
|
+
component: "scrapers",
|
|
17168
|
+
operation: "pdf-extract",
|
|
17169
|
+
contentType: "pdf",
|
|
17170
|
+
errorType: "native_module_unavailable"
|
|
17171
|
+
});
|
|
17172
|
+
throw new Error(`PDF extraction unavailable (pdf-oxide-wasm failed to load: ${msg2})`, { cause: e });
|
|
17173
|
+
}
|
|
17100
17174
|
try {
|
|
17101
|
-
const { WasmPdfDocument } = await import("pdf-oxide-wasm");
|
|
17102
17175
|
const doc = new WasmPdfDocument(bytes);
|
|
17103
17176
|
const pageCount = doc.pageCount();
|
|
17104
17177
|
let markdown = `# PDF Document
|
|
@@ -18548,9 +18621,9 @@ async function searchGitHubAdvisories(terms, options) {
|
|
|
18548
18621
|
throw new Error(`Invalid repo format: "${options.repo}". Expected "owner/name".`);
|
|
18549
18622
|
}
|
|
18550
18623
|
const [owner, name] = repoParts;
|
|
18551
|
-
const repoNameRe = /^[A-Za-z0-9._-]+$/;
|
|
18624
|
+
const repoNameRe = /^(?!\.+$)[A-Za-z0-9._-]+$/;
|
|
18552
18625
|
if (!repoNameRe.test(owner) || !repoNameRe.test(name)) {
|
|
18553
|
-
throw new Error(`Invalid repo format: "${options.repo}". Owner and name may only contain letters, digits, '.', '_' and '-'.`);
|
|
18626
|
+
throw new Error(`Invalid repo format: "${options.repo}". Owner and name may only contain letters, digits, '.', '_' and '-', and cannot be all dots.`);
|
|
18554
18627
|
}
|
|
18555
18628
|
const url = `${GITHUB_API_BASE}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/security-advisories?per_page=${perPage}`;
|
|
18556
18629
|
const response = await githubCircuitBreaker.execute(() => retryWithBackoff(async () => {
|
|
@@ -19674,6 +19747,7 @@ init_logger();
|
|
|
19674
19747
|
init_circuit_breaker();
|
|
19675
19748
|
init_metrics();
|
|
19676
19749
|
init_http_body();
|
|
19750
|
+
init_retry_utils();
|
|
19677
19751
|
var API_BASE = "https://api.stackexchange.com/2.3";
|
|
19678
19752
|
var QUOTA_PROBE_COOLDOWN_MS = 10 * 60 * 1e3;
|
|
19679
19753
|
var StackExchangeClient = class {
|
|
@@ -19721,21 +19795,11 @@ var StackExchangeClient = class {
|
|
|
19721
19795
|
if (!url.searchParams.has("site") && !options.endpoint.startsWith("/sites")) {
|
|
19722
19796
|
url.searchParams.set("site", "stackoverflow.com");
|
|
19723
19797
|
}
|
|
19724
|
-
const
|
|
19725
|
-
const timeoutId = setTimeout(() => {
|
|
19726
|
-
controller.abort();
|
|
19727
|
-
}, this._timeout);
|
|
19728
|
-
const abortHandler = () => {
|
|
19729
|
-
clearTimeout(timeoutId);
|
|
19730
|
-
controller.abort();
|
|
19731
|
-
};
|
|
19732
|
-
if (signal) {
|
|
19733
|
-
signal.addEventListener("abort", abortHandler, { once: true });
|
|
19734
|
-
}
|
|
19798
|
+
const requestSignal = createTimeoutSignal(this._timeout, signal);
|
|
19735
19799
|
try {
|
|
19736
19800
|
const response = await fetch(url.toString(), {
|
|
19737
19801
|
method: options.method,
|
|
19738
|
-
signal:
|
|
19802
|
+
signal: requestSignal,
|
|
19739
19803
|
headers: {
|
|
19740
19804
|
"Accept": "application/json"
|
|
19741
19805
|
}
|
|
@@ -19790,11 +19854,6 @@ var StackExchangeClient = class {
|
|
|
19790
19854
|
throw new Error(`Request timeout after ${this._timeout}ms`, { cause: error });
|
|
19791
19855
|
}
|
|
19792
19856
|
throw error;
|
|
19793
|
-
} finally {
|
|
19794
|
-
clearTimeout(timeoutId);
|
|
19795
|
-
if (signal) {
|
|
19796
|
-
signal.removeEventListener("abort", abortHandler);
|
|
19797
|
-
}
|
|
19798
19857
|
}
|
|
19799
19858
|
});
|
|
19800
19859
|
}
|
|
@@ -20035,6 +20094,33 @@ function formatCompactQuestions(questions) {
|
|
|
20035
20094
|
}
|
|
20036
20095
|
|
|
20037
20096
|
// src/stackexchange/output/compact.ts
|
|
20097
|
+
function formatQuestionsCompact(questions) {
|
|
20098
|
+
if (questions.length === 0) {
|
|
20099
|
+
return "No questions found.";
|
|
20100
|
+
}
|
|
20101
|
+
const lines = [];
|
|
20102
|
+
let index = 1;
|
|
20103
|
+
for (const q of questions) {
|
|
20104
|
+
const accepted = q.accepted_answer_id ? "[accepted]" : " ";
|
|
20105
|
+
lines.push(`${index}. ${accepted} [${q.title}](${q.link}) (${q.score} pts, ${q.answer_count} ans, ${q.view_count} views)`);
|
|
20106
|
+
index++;
|
|
20107
|
+
}
|
|
20108
|
+
return lines.join("\n");
|
|
20109
|
+
}
|
|
20110
|
+
function formatAnswersCompact(answers) {
|
|
20111
|
+
if (answers.length === 0) {
|
|
20112
|
+
return "No answers found.";
|
|
20113
|
+
}
|
|
20114
|
+
const lines = [];
|
|
20115
|
+
let index = 1;
|
|
20116
|
+
for (const a of answers) {
|
|
20117
|
+
const accepted = a.is_accepted ? "[accepted]" : " ";
|
|
20118
|
+
const displayName = a.owner?.display_name ?? "Unknown";
|
|
20119
|
+
lines.push(`${index}. ${accepted} by ${displayName} (${a.score} pts)`);
|
|
20120
|
+
index++;
|
|
20121
|
+
}
|
|
20122
|
+
return lines.join("\n");
|
|
20123
|
+
}
|
|
20038
20124
|
function formatUsersCompact(users) {
|
|
20039
20125
|
if (users.length === 0) {
|
|
20040
20126
|
return "No users found.";
|
|
@@ -20318,6 +20404,16 @@ function formatCompact(result) {
|
|
|
20318
20404
|
return formatCompactQuestions(result);
|
|
20319
20405
|
}
|
|
20320
20406
|
}
|
|
20407
|
+
} else if (typeof result === "object" && result !== null) {
|
|
20408
|
+
if ("question" in result && "answers" in result) {
|
|
20409
|
+
const r = result;
|
|
20410
|
+
let output = formatQuestionsCompact([r.question]);
|
|
20411
|
+
if (r.answers.length > 0) {
|
|
20412
|
+
output += `
|
|
20413
|
+
${formatAnswersCompact(r.answers)}`;
|
|
20414
|
+
}
|
|
20415
|
+
return output;
|
|
20416
|
+
}
|
|
20321
20417
|
}
|
|
20322
20418
|
return JSON.stringify(result);
|
|
20323
20419
|
}
|
|
@@ -22362,32 +22458,36 @@ ${result}`);
|
|
|
22362
22458
|
if (this.config.KNOWLEDGE_STORE_MODE !== "none") {
|
|
22363
22459
|
const ksService = await getService(ServiceNames.KNOWLEDGE_STORE, ctx, container);
|
|
22364
22460
|
if (ksService.isReady()) {
|
|
22365
|
-
const writer = await
|
|
22366
|
-
|
|
22367
|
-
|
|
22368
|
-
|
|
22369
|
-
|
|
22370
|
-
|
|
22371
|
-
|
|
22372
|
-
if (cit.url) {
|
|
22373
|
-
const fullContent = getCachedScrapedContent(this.options.researchId, cit.url);
|
|
22374
|
-
const markdown = cit.description || `(source: ${cit.url})`;
|
|
22375
|
-
writer.enqueue({
|
|
22376
|
-
url: normalizeUrl(cit.url),
|
|
22377
|
-
markdown,
|
|
22378
|
-
content: fullContent,
|
|
22379
|
-
metadata: {
|
|
22380
|
-
ingestionType: "synthesis-description",
|
|
22381
|
-
source: "researcher",
|
|
22382
|
-
synthesizedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
22383
|
-
description: cit.description || "",
|
|
22384
|
-
fullContentSnippet: fullContent?.substring(0, 5e3)
|
|
22385
|
-
}
|
|
22386
|
-
});
|
|
22387
|
-
enqueued++;
|
|
22461
|
+
const writer = await ksService.getWriterQueue();
|
|
22462
|
+
if (!writer) {
|
|
22463
|
+
logger.debug("[QuickOrchestrator] Writer queue unavailable, skipping link descriptions");
|
|
22464
|
+
} else {
|
|
22465
|
+
const citations = parseCitations(result);
|
|
22466
|
+
if (citations.length === 0) {
|
|
22467
|
+
logger.warn("[QuickOrchestrator] Researcher produced no parseable CITED LINKS \u2014 no descriptions stored for this session");
|
|
22388
22468
|
}
|
|
22469
|
+
let enqueued = 0;
|
|
22470
|
+
for (const cit of citations) {
|
|
22471
|
+
if (cit.url) {
|
|
22472
|
+
const fullContent = getCachedScrapedContent(this.options.researchId, cit.url);
|
|
22473
|
+
const markdown = cit.description || `(source: ${cit.url})`;
|
|
22474
|
+
writer.enqueue({
|
|
22475
|
+
url: normalizeUrl(cit.url),
|
|
22476
|
+
markdown,
|
|
22477
|
+
content: fullContent,
|
|
22478
|
+
metadata: {
|
|
22479
|
+
ingestionType: "synthesis-description",
|
|
22480
|
+
source: "researcher",
|
|
22481
|
+
synthesizedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
22482
|
+
description: cit.description || "",
|
|
22483
|
+
fullContentSnippet: fullContent?.substring(0, 5e3)
|
|
22484
|
+
}
|
|
22485
|
+
});
|
|
22486
|
+
enqueued++;
|
|
22487
|
+
}
|
|
22488
|
+
}
|
|
22489
|
+
if (enqueued > 0) await writer.drain();
|
|
22389
22490
|
}
|
|
22390
|
-
if (enqueued > 0) await writer.drain();
|
|
22391
22491
|
}
|
|
22392
22492
|
}
|
|
22393
22493
|
} catch (err) {
|
|
@@ -23180,7 +23280,11 @@ var ResearchOrchestrationService = class {
|
|
|
23180
23280
|
return;
|
|
23181
23281
|
}
|
|
23182
23282
|
const synthesisService = await getService(ServiceNames.RESEARCH_SYNTHESIS_SERVICE, ctx, container);
|
|
23183
|
-
const writer = await
|
|
23283
|
+
const writer = await ksService.getWriterQueue();
|
|
23284
|
+
if (!writer) {
|
|
23285
|
+
logger.debug("[ResearchOrchestrationService] Writer queue unavailable, skipping link descriptions");
|
|
23286
|
+
return;
|
|
23287
|
+
}
|
|
23184
23288
|
const roundPrefix = `${round}.`;
|
|
23185
23289
|
let enqueued = 0;
|
|
23186
23290
|
let researcherCount = 0;
|
|
@@ -23833,6 +23937,7 @@ async function initializeOrchestrationServices(ctx, container = getServiceContai
|
|
|
23833
23937
|
init_service_registry();
|
|
23834
23938
|
init_service_interfaces();
|
|
23835
23939
|
init_logger();
|
|
23940
|
+
init_log_utils();
|
|
23836
23941
|
init_config();
|
|
23837
23942
|
init_metrics();
|
|
23838
23943
|
|
|
@@ -24175,7 +24280,7 @@ function getLastResearcherOutcome() {
|
|
|
24175
24280
|
}
|
|
24176
24281
|
function logRunErrorSummary(report, depthLabel, status) {
|
|
24177
24282
|
if (!report || report.totalErrors === 0) return;
|
|
24178
|
-
const top = report.patterns.slice(0, 3).map((p) => `"${String(p.message ?? p.signature ?? "error").slice(0, 48)}" \xD7${p.count}`).join(", ");
|
|
24283
|
+
const top = report.patterns.slice(0, 3).map((p) => `"${redactSecrets(String(p.message ?? p.signature ?? "error")).slice(0, 48)}" \xD7${p.count}`).join(", ");
|
|
24179
24284
|
logger.warn(`[SDK] run ${status} with ${report.totalErrors} tracked error(s) across ${report.uniquePatterns} pattern(s) [${depthLabel}]${top ? ` \u2014 top: ${top}` : ""}`);
|
|
24180
24285
|
}
|
|
24181
24286
|
function wrapObserverForOutcomeCapture(observer, onPlanned) {
|