@lincoln504/pi-research 1.3.3 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.mjs +177 -72
- package/dist/thread-worker.mjs +56 -37
- package/docs/CHANGELOG.md +39 -0
- package/package.json +1 -1
- package/skills/pi-research/SKILL.md +1 -1
- package/src/core/service-registry.ts +37 -3
- package/src/healthcheck/index.ts +9 -7
- package/src/infrastructure/browser/task-execution-service.ts +21 -5
- package/src/infrastructure/browser/thread-worker-browser.ts +20 -1
- package/src/infrastructure/browser/thread-worker-messaging.ts +30 -22
- package/src/infrastructure/browser/thread-worker.mjs +56 -37
- package/src/infrastructure/knowledge-store-service.ts +27 -2
- package/src/knowledge/chunker.ts +14 -1
- package/src/orchestration/quick-research-orchestrator.ts +13 -2
- package/src/orchestration/research-orchestration-service.ts +16 -3
- package/src/sdk.ts +11 -2
- package/src/security/github-advisories.ts +13 -4
- package/src/skill-install/skill-installer.ts +40 -5
- package/src/stackexchange/index.ts +15 -0
- package/src/stackexchange/rest-client.ts +13 -24
- package/src/utils/input-validation.ts +0 -10
- package/src/utils/json-utils.ts +0 -78
- package/src/utils/log-utils.ts +32 -33
- package/src/utils/metrics.ts +0 -8
- package/src/utils/shared-links.ts +0 -14
- package/src/utils/stdio-capture.ts +44 -7
- package/src/utils/url-utils.ts +7 -1
- package/src/web-research/scraper-utils.ts +8 -3
- package/src/web-research/web-scraper.ts +33 -1
|
@@ -45,12 +45,7 @@ function formatArg(arg) {
|
|
|
45
45
|
return String(arg);
|
|
46
46
|
}
|
|
47
47
|
function redactSecrets(message) {
|
|
48
|
-
let
|
|
49
|
-
if (message.length > keep) {
|
|
50
|
-
const lastKeptUnit = message.charCodeAt(keep - 1);
|
|
51
|
-
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
52
|
-
}
|
|
53
|
-
let out = message.length > MAX_LOG_MESSAGE_LENGTH ? `${message.slice(0, keep)}\u2026[truncated ${message.length - keep} chars]` : message;
|
|
48
|
+
let out = message;
|
|
54
49
|
out = out.replace(ANSI_PATTERN, "");
|
|
55
50
|
out = out.replace(URL_CREDENTIALS_PATTERN, "$1[REDACTED]@");
|
|
56
51
|
out = out.replace(JWT_PATTERN, "[REDACTED]");
|
|
@@ -61,6 +56,14 @@ function redactSecrets(message) {
|
|
|
61
56
|
out = out.replace(KNOWN_TOKEN_PATTERN, "[REDACTED]");
|
|
62
57
|
out = out.replace(PROVIDER_TOKEN_PATTERN, "[REDACTED]");
|
|
63
58
|
out = out.replace(LONG_HEX_SECRET_PATTERN, "[REDACTED]");
|
|
59
|
+
let keep = MAX_LOG_MESSAGE_LENGTH;
|
|
60
|
+
if (out.length > keep) {
|
|
61
|
+
const lastKeptUnit = out.charCodeAt(keep - 1);
|
|
62
|
+
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
63
|
+
}
|
|
64
|
+
if (out.length > MAX_LOG_MESSAGE_LENGTH) {
|
|
65
|
+
out = `${out.slice(0, keep)}\u2026[truncated ${out.length - keep} chars]`;
|
|
66
|
+
}
|
|
64
67
|
return out;
|
|
65
68
|
}
|
|
66
69
|
function stripTerminalEscapes(message) {
|
|
@@ -75,7 +78,7 @@ var init_log_utils = __esm({
|
|
|
75
78
|
ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
|
|
76
79
|
URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
|
|
77
80
|
COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
|
|
78
|
-
SENSITIVE_KV_PATTERN = /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
|
|
81
|
+
SENSITIVE_KV_PATTERN = /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
|
|
79
82
|
KNOWN_TOKEN_PATTERN = /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
|
|
80
83
|
JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
|
|
81
84
|
BASIC_AUTH_PATTERN = /\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/gi;
|
|
@@ -588,7 +591,10 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
588
591
|
unkeyedCaptureDepth++;
|
|
589
592
|
isAnyLoggerCapturingOutput = true;
|
|
590
593
|
}
|
|
591
|
-
const
|
|
594
|
+
const stderrDecoder = new TextDecoder();
|
|
595
|
+
const stdoutDecoder = new TextDecoder();
|
|
596
|
+
const fd1SyncDecoder = new TextDecoder();
|
|
597
|
+
const fd2SyncDecoder = new TextDecoder();
|
|
592
598
|
const originalStderrWrite = process.stderr.write;
|
|
593
599
|
const originalStdoutWrite = process.stdout.write;
|
|
594
600
|
const originalFsWriteSync = fs4.writeSync;
|
|
@@ -634,7 +640,11 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
634
640
|
console.debug = createConsolePatch("debug", logFile, hasSufficientDiskSpace);
|
|
635
641
|
process.stderr.write = (chunk, encodingOrCb, callback) => {
|
|
636
642
|
const cb = typeof encodingOrCb === "function" ? encodingOrCb : callback;
|
|
637
|
-
const message = typeof chunk === "string" ? chunk :
|
|
643
|
+
const message = typeof chunk === "string" ? chunk : stderrDecoder.decode(chunk, { stream: true });
|
|
644
|
+
if (message.length === 0) {
|
|
645
|
+
if (typeof cb === "function") cb();
|
|
646
|
+
return true;
|
|
647
|
+
}
|
|
638
648
|
if (!hasSufficientDiskSpace()) {
|
|
639
649
|
if (typeof cb === "function") cb();
|
|
640
650
|
return true;
|
|
@@ -655,7 +665,7 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
655
665
|
return true;
|
|
656
666
|
};
|
|
657
667
|
process.stdout.write = (chunk, encodingOrCb, callback) => {
|
|
658
|
-
const message = typeof chunk === "string" ? chunk :
|
|
668
|
+
const message = typeof chunk === "string" ? chunk : stdoutDecoder.decode(chunk, { stream: true });
|
|
659
669
|
const isNative = isNativeLog(message);
|
|
660
670
|
const complexTui = isComplexTui(message);
|
|
661
671
|
const boxDrawing = isBoxDrawing(message);
|
|
@@ -709,7 +719,11 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
709
719
|
if (!descriptor || (descriptor.writable || descriptor.set)) {
|
|
710
720
|
fs4.writeSync = (fd, chunk, ...args) => {
|
|
711
721
|
if (fd === 1 || fd === 2) {
|
|
712
|
-
const
|
|
722
|
+
const fdDecoder = fd === 1 ? fd1SyncDecoder : fd2SyncDecoder;
|
|
723
|
+
const message = typeof chunk === "string" ? chunk : fdDecoder.decode(chunk, { stream: true });
|
|
724
|
+
if (fd === 2 && message.length === 0) {
|
|
725
|
+
return typeof chunk === "string" ? Buffer.from(chunk).length : chunk.length;
|
|
726
|
+
}
|
|
713
727
|
const isNative = isNativeLog(message);
|
|
714
728
|
const boxDrawing = isBoxDrawing(message);
|
|
715
729
|
const complexTui = isComplexTui(message);
|
|
@@ -1489,6 +1503,7 @@ function isBenignScrapeFailure(error) {
|
|
|
1489
1503
|
return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
|
|
1490
1504
|
msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
|
|
1491
1505
|
msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
|
|
1506
|
+
msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
|
|
1492
1507
|
/^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
|
|
1493
1508
|
isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
|
|
1494
1509
|
}
|
|
@@ -1914,32 +1929,29 @@ async function createPageSafe(context2) {
|
|
|
1914
1929
|
const currentLock = pageCreationLock;
|
|
1915
1930
|
pageCreationLock = currentLock.then(() => nextLock);
|
|
1916
1931
|
await currentLock;
|
|
1932
|
+
const pagePromise = context2.newPage();
|
|
1933
|
+
pagePromise.then(release, release);
|
|
1934
|
+
pagePromise.catch((err) => logToDebugFile2("DEBUG", `[ThreadWorker] Background page creation rejection: ${err.message}`));
|
|
1935
|
+
let timeoutId;
|
|
1917
1936
|
try {
|
|
1918
|
-
const
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
}
|
|
1937
|
-
}).catch(() => {
|
|
1938
|
-
});
|
|
1939
|
-
throw err;
|
|
1940
|
-
}
|
|
1941
|
-
} finally {
|
|
1942
|
-
release();
|
|
1937
|
+
const result = await Promise.race([
|
|
1938
|
+
pagePromise,
|
|
1939
|
+
new Promise((_, reject) => {
|
|
1940
|
+
timeoutId = setTimeout(() => reject(new Error("Browser page creation timed out after 60000ms")), 6e4);
|
|
1941
|
+
})
|
|
1942
|
+
]);
|
|
1943
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
1944
|
+
return result;
|
|
1945
|
+
} catch (err) {
|
|
1946
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
1947
|
+
pagePromise.then((p) => {
|
|
1948
|
+
try {
|
|
1949
|
+
p?.close?.();
|
|
1950
|
+
} catch {
|
|
1951
|
+
}
|
|
1952
|
+
}).catch(() => {
|
|
1953
|
+
});
|
|
1954
|
+
throw err;
|
|
1943
1955
|
}
|
|
1944
1956
|
}
|
|
1945
1957
|
function setWorkerId2(id) {
|
|
@@ -2339,6 +2351,7 @@ init_logger();
|
|
|
2339
2351
|
|
|
2340
2352
|
// src/utils/url-utils.ts
|
|
2341
2353
|
init_logger();
|
|
2354
|
+
init_log_utils();
|
|
2342
2355
|
|
|
2343
2356
|
// src/config.ts
|
|
2344
2357
|
var ConfigSchema = Type.Object({
|
|
@@ -2508,6 +2521,7 @@ var initPromise = null;
|
|
|
2508
2521
|
var workerId3 = "";
|
|
2509
2522
|
var activeTaskCount = 0;
|
|
2510
2523
|
var resetPending = false;
|
|
2524
|
+
var resetPendingForBrowser = null;
|
|
2511
2525
|
function setWorkerId3(id) {
|
|
2512
2526
|
workerId3 = id;
|
|
2513
2527
|
}
|
|
@@ -2518,7 +2532,11 @@ function taskFinished() {
|
|
|
2518
2532
|
activeTaskCount = Math.max(0, activeTaskCount - 1);
|
|
2519
2533
|
if (activeTaskCount === 0 && resetPending) {
|
|
2520
2534
|
resetPending = false;
|
|
2521
|
-
|
|
2535
|
+
const capturedBrowser = resetPendingForBrowser;
|
|
2536
|
+
resetPendingForBrowser = null;
|
|
2537
|
+
if (browser === capturedBrowser) {
|
|
2538
|
+
doResetBrowser();
|
|
2539
|
+
}
|
|
2522
2540
|
}
|
|
2523
2541
|
}
|
|
2524
2542
|
var isBrowserConnected = () => {
|
|
@@ -2655,6 +2673,7 @@ function doResetBrowser() {
|
|
|
2655
2673
|
function resetBrowser() {
|
|
2656
2674
|
if (activeTaskCount > 1) {
|
|
2657
2675
|
resetPending = true;
|
|
2676
|
+
resetPendingForBrowser = browser;
|
|
2658
2677
|
return;
|
|
2659
2678
|
}
|
|
2660
2679
|
doResetBrowser();
|
|
@@ -311,6 +311,17 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
|
|
|
311
311
|
} catch {
|
|
312
312
|
// init failed; its own catch already reset state — nothing to wait on.
|
|
313
313
|
}
|
|
314
|
+
// Re-check: a SECOND concurrent dispose() call can reach this same point
|
|
315
|
+
// (its own _disposalPromise check above also saw null, before either of
|
|
316
|
+
// us had published one) and, if it settles the await first, publish its
|
|
317
|
+
// own disposal while we were still awaiting. Without this, both calls
|
|
318
|
+
// would go on to capture the SAME embedder/store/writerQueue/initLock
|
|
319
|
+
// references below and each independently call .dispose()/.close() on
|
|
320
|
+
// them — the same double-teardown-of-one-instance class of bug fixed at
|
|
321
|
+
// the ServiceContainer level (disposeAll() vs clear()/replace()).
|
|
322
|
+
if (this._disposalPromise) {
|
|
323
|
+
return this._disposalPromise;
|
|
324
|
+
}
|
|
314
325
|
}
|
|
315
326
|
|
|
316
327
|
this.lifecycle = ServiceLifecycle.DISPOSING;
|
|
@@ -330,7 +341,10 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
|
|
|
330
341
|
const writerQueue = this._writerQueue;
|
|
331
342
|
const initLock = this._initLock;
|
|
332
343
|
|
|
333
|
-
|
|
344
|
+
// Named locally (mirrors initialize()'s initPromise) so the finally below
|
|
345
|
+
// can identity-check before clearing — see there for why.
|
|
346
|
+
let disposePromise: Promise<void> | null = null;
|
|
347
|
+
disposePromise = (async () => {
|
|
334
348
|
try {
|
|
335
349
|
if (writerQueue) {
|
|
336
350
|
await writerQueue.dispose?.();
|
|
@@ -370,9 +384,20 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
|
|
|
370
384
|
logger.error('[KnowledgeStoreService] Error during disposal:', err);
|
|
371
385
|
} finally {
|
|
372
386
|
this.lifecycle = ServiceLifecycle.DISPOSED;
|
|
373
|
-
|
|
387
|
+
// Identity-guarded, mirroring initialize()'s own marker clear (see its
|
|
388
|
+
// finally): only the disposal that still owns _disposalPromise may
|
|
389
|
+
// null it. The re-check above closes the common race, but this is the
|
|
390
|
+
// same defense-in-depth initialize() uses — without it, a stale
|
|
391
|
+
// closure's completion could null a NEWER disposal's still-in-flight
|
|
392
|
+
// marker, making a concurrent initialize() believe nothing is
|
|
393
|
+
// disposing and rebuild components while that newer disposal is still
|
|
394
|
+
// tearing them down.
|
|
395
|
+
if (this._disposalPromise === disposePromise) {
|
|
396
|
+
this._disposalPromise = null;
|
|
397
|
+
}
|
|
374
398
|
}
|
|
375
399
|
})();
|
|
400
|
+
this._disposalPromise = disposePromise;
|
|
376
401
|
|
|
377
402
|
return this._disposalPromise;
|
|
378
403
|
}
|
package/src/knowledge/chunker.ts
CHANGED
|
@@ -209,12 +209,25 @@ export class Chunker {
|
|
|
209
209
|
// pair's high half is guaranteed already present in the chunk just
|
|
210
210
|
// pushed, so skipping the low half here loses nothing, it just narrows
|
|
211
211
|
// this chunk's overlap by one code unit instead of widening it.
|
|
212
|
+
//
|
|
213
|
+
// The push-forward bound is `<= end`, not `< end`: landing exactly on
|
|
214
|
+
// `end` (start === end, zero overlap for this transition) is a
|
|
215
|
+
// perfectly safe position — `end` is always a valid index up to
|
|
216
|
+
// text.length, and the NEXT chunk's own bounds are recomputed fresh
|
|
217
|
+
// from `start` at the top of the loop, independent of this `end`. A
|
|
218
|
+
// strict `<` excluded that boundary case, so at tight targetSize/
|
|
219
|
+
// overlap ratios (e.g. targetSize:2, overlap:1, straddling a surrogate
|
|
220
|
+
// pair) BOTH escapes could be simultaneously blocked — the pull-back
|
|
221
|
+
// correctly refused (it would violate forward-progress) and the
|
|
222
|
+
// push-forward wrongly refused too (start+1 landed exactly on `end`) —
|
|
223
|
+
// leaving `start` unmoved on the lone low surrogate, corrupting the
|
|
224
|
+
// next chunk's opening character.
|
|
212
225
|
if (start > 0) {
|
|
213
226
|
const code = text.charCodeAt(start);
|
|
214
227
|
if (code >= 0xdc00 && code <= 0xdfff) {
|
|
215
228
|
if (start - 1 > iterationStart) {
|
|
216
229
|
start -= 1;
|
|
217
|
-
} else if (start + 1
|
|
230
|
+
} else if (start + 1 <= end) {
|
|
218
231
|
start += 1;
|
|
219
232
|
}
|
|
220
233
|
}
|
|
@@ -22,7 +22,7 @@ import { getMaxScrapeBatches, resolveExcludedTools } from '../constants.ts';
|
|
|
22
22
|
import type { ResearchObserver } from '../core/interfaces/observer-interfaces.ts';
|
|
23
23
|
import { HeadlessObserver, makeSafeObserver, type HeadlessObserverOptions } from './headless-observer.ts';
|
|
24
24
|
import { getService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
|
|
25
|
-
import { ServiceNames, type
|
|
25
|
+
import { ServiceNames, type IKnowledgeStoreService, type IResearchSynthesisService, type IResearchOrchestration } from '../core/service-interfaces.ts';
|
|
26
26
|
import type { ResearchSessionService } from './research-session-service.ts';
|
|
27
27
|
import { normalizeUrl, registerScrapedLinks, getCachedScrapedContent } from '../utils/shared-links.ts';
|
|
28
28
|
import { runHealthCheck, isBusyPoolHealthFailure } from '../healthcheck/index.ts';
|
|
@@ -347,7 +347,17 @@ export class QuickResearchOrchestrator {
|
|
|
347
347
|
if (this.config.KNOWLEDGE_STORE_MODE !== 'none') {
|
|
348
348
|
const ksService = await getService<IKnowledgeStoreService>(ServiceNames.KNOWLEDGE_STORE, ctx, container);
|
|
349
349
|
if (ksService.isReady()) {
|
|
350
|
-
|
|
350
|
+
// Resolve directly from ksService rather than the WRITER_QUEUE DI
|
|
351
|
+
// registration — that registration caches the first IWriterQueue it
|
|
352
|
+
// is ever handed forever, so it goes stale after a Knowledge
|
|
353
|
+
// Mode/cwd change disposes and rebuilds ksService's internal writer
|
|
354
|
+
// queue (see research-orchestration-service.ts's storeLinkDescriptions
|
|
355
|
+
// for the full explanation). ksService.getWriterQueue() always
|
|
356
|
+
// reflects what initialize() just settled on.
|
|
357
|
+
const writer = await ksService.getWriterQueue();
|
|
358
|
+
if (!writer) {
|
|
359
|
+
logger.debug('[QuickOrchestrator] Writer queue unavailable, skipping link descriptions');
|
|
360
|
+
} else {
|
|
351
361
|
const citations = parseCitations(result);
|
|
352
362
|
if (citations.length === 0) {
|
|
353
363
|
logger.warn('[QuickOrchestrator] Researcher produced no parseable CITED LINKS — no descriptions stored for this session');
|
|
@@ -375,6 +385,7 @@ export class QuickResearchOrchestrator {
|
|
|
375
385
|
// Drain so concurrent or subsequent sessions see these entries immediately
|
|
376
386
|
// rather than relying solely on shutdownKnowledgeStore's drain.
|
|
377
387
|
if (enqueued > 0) await writer.drain();
|
|
388
|
+
}
|
|
378
389
|
}
|
|
379
390
|
}
|
|
380
391
|
} catch (err) {
|
|
@@ -23,7 +23,6 @@ import { clearSessionCircuitBreaker } from '../infrastructure/browser/browser-er
|
|
|
23
23
|
import { getService, ServiceLifecycle, tryGetService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
|
|
24
24
|
import { ServiceNames } from '../core/service-interfaces.ts';
|
|
25
25
|
import type {
|
|
26
|
-
IWriterQueue,
|
|
27
26
|
IKnowledgeStoreService,
|
|
28
27
|
IHealthRegistryService,
|
|
29
28
|
IResearchOrchestration,
|
|
@@ -548,8 +547,22 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
548
547
|
}
|
|
549
548
|
|
|
550
549
|
const synthesisService = await getService<IResearchSynthesisService>(ServiceNames.RESEARCH_SYNTHESIS_SERVICE, ctx, container);
|
|
551
|
-
|
|
552
|
-
|
|
550
|
+
// Resolve the writer queue directly from the already-fresh ksService rather
|
|
551
|
+
// than through the WRITER_QUEUE DI registration: that registration caches the
|
|
552
|
+
// FIRST IWriterQueue object it is ever handed, forever — WriterQueue's own
|
|
553
|
+
// initialize() is a no-arg lifecycle no-op that never rebuilds its bound
|
|
554
|
+
// store/chunker, so a later Knowledge Mode/cwd change that disposes and
|
|
555
|
+
// rebuilds ksService's internal writer queue left every future getService()
|
|
556
|
+
// call here still returning the stale, disposed one (writes silently dropped
|
|
557
|
+
// via its "store is closing" guard). ksService.getWriterQueue() always
|
|
558
|
+
// reflects whatever initialize() just settled on, which the isReady() check
|
|
559
|
+
// above already confirmed is live.
|
|
560
|
+
const writer = await ksService.getWriterQueue();
|
|
561
|
+
if (!writer) {
|
|
562
|
+
logger.debug('[ResearchOrchestrationService] Writer queue unavailable, skipping link descriptions');
|
|
563
|
+
return;
|
|
564
|
+
}
|
|
565
|
+
|
|
553
566
|
const roundPrefix = `${round}.`;
|
|
554
567
|
let enqueued = 0;
|
|
555
568
|
let researcherCount = 0;
|
package/src/sdk.ts
CHANGED
|
@@ -34,6 +34,7 @@ import { HeadlessObserver } from './orchestration/headless-observer.ts';
|
|
|
34
34
|
import type { Model } from '@earendil-works/pi-ai';
|
|
35
35
|
import { type ExtensionContext, ModelRegistry } from '@earendil-works/pi-coding-agent';
|
|
36
36
|
import { logger, createLogger, setLogger } from './logger.ts';
|
|
37
|
+
import { redactSecrets } from './utils/log-utils.ts';
|
|
37
38
|
import { getConfig, createConfig, validateConfig, resetConfig, type Config } from './config.ts';
|
|
38
39
|
import { metrics, MetricsRegistry, runWithRunRegistry } from './utils/metrics.ts';
|
|
39
40
|
import type { IMetricsSnapshot, RunSummary } from './utils/metrics.ts';
|
|
@@ -633,10 +634,18 @@ export function getLastResearcherOutcome(): ResearcherOutcome | null {
|
|
|
633
634
|
* Emit a single compact line summarizing a run's tracked errors, so they are visible in
|
|
634
635
|
* the operator's log instead of silently accumulating. No-op when the run was clean.
|
|
635
636
|
*/
|
|
636
|
-
function logRunErrorSummary(report: ErrorReport | null, depthLabel: string, status: 'success' | 'error'): void {
|
|
637
|
+
export function logRunErrorSummary(report: ErrorReport | null, depthLabel: string, status: 'success' | 'error'): void {
|
|
637
638
|
if (!report || report.totalErrors === 0) return;
|
|
639
|
+
// Redact BEFORE truncating to 48 chars: p.message is the RAW, unredacted
|
|
640
|
+
// first-seen error message (unlike p.signature, which extractSignature()
|
|
641
|
+
// already strips URLs from) — a scrape/navigation failure routinely echoes
|
|
642
|
+
// the failing URL verbatim, which can carry userinfo credentials
|
|
643
|
+
// (https://user:pass@host/...). A 48-char window makes landing mid-
|
|
644
|
+
// credential likely; truncating first would cut it before redactSecrets'
|
|
645
|
+
// patterns ever see the whole token. Same class of bug as the truncate-
|
|
646
|
+
// before-redact fix in log-utils.ts's redactSecrets itself.
|
|
638
647
|
const top = report.patterns.slice(0, 3)
|
|
639
|
-
.map(p => `"${String(p.message ?? p.signature ?? 'error').slice(0, 48)}" ×${p.count}`)
|
|
648
|
+
.map(p => `"${redactSecrets(String(p.message ?? p.signature ?? 'error')).slice(0, 48)}" ×${p.count}`)
|
|
640
649
|
.join(', ');
|
|
641
650
|
logger.warn(`[SDK] run ${status} with ${report.totalErrors} tracked error(s) across ${report.uniquePatterns} pattern(s) [${depthLabel}]${top ? ` — top: ${top}` : ''}`);
|
|
642
651
|
}
|
|
@@ -102,11 +102,20 @@ export async function searchGitHubAdvisories(
|
|
|
102
102
|
|
|
103
103
|
const [owner, name] = repoParts;
|
|
104
104
|
// Constrain to GitHub's allowed owner/repo character set before interpolating
|
|
105
|
-
// into the API path. encodeURIComponent
|
|
106
|
-
//
|
|
107
|
-
|
|
105
|
+
// into the API path. encodeURIComponent does NOT neutralise traversal here:
|
|
106
|
+
// '.' and '-' are unreserved characters it passes through unescaped, so an
|
|
107
|
+
// owner/name segment of exactly '.' or '..' survives encodeURIComponent
|
|
108
|
+
// intact and is then collapsed by URL/fetch's own path normalization BEFORE
|
|
109
|
+
// the request is sent — e.g. owner=".." turns "/repos/../rate_limit/..."
|
|
110
|
+
// into a request for "/rate_limit/...", escaping the intended
|
|
111
|
+
// /repos/{owner}/{name}/ scope entirely (verified against the real URL
|
|
112
|
+
// parser). The negative lookahead rejects a segment that is ENTIRELY dots
|
|
113
|
+
// (the only shape path normalization treats as a traversal token) while
|
|
114
|
+
// still allowing legitimate names that merely contain dots (node.js,
|
|
115
|
+
// socket.io).
|
|
116
|
+
const repoNameRe = /^(?!\.+$)[A-Za-z0-9._-]+$/;
|
|
108
117
|
if (!repoNameRe.test(owner!) || !repoNameRe.test(name!)) {
|
|
109
|
-
throw new Error(`Invalid repo format: "${options.repo}". Owner and name may only contain letters, digits, '.', '_' and '-'.`);
|
|
118
|
+
throw new Error(`Invalid repo format: "${options.repo}". Owner and name may only contain letters, digits, '.', '_' and '-', and cannot be all dots.`);
|
|
110
119
|
}
|
|
111
120
|
const url = `${GITHUB_API_BASE}/repos/${encodeURIComponent(owner!)}/${encodeURIComponent(name!)}/security-advisories?per_page=${perPage}`;
|
|
112
121
|
|
|
@@ -374,15 +374,50 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
|
|
|
374
374
|
continue;
|
|
375
375
|
}
|
|
376
376
|
fs.unlinkSync(absSkillPath);
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
377
|
+
try {
|
|
378
|
+
fs.symlinkSync(source, absSkillPath, process.platform === 'win32' ? 'junction' : 'dir');
|
|
379
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'symlink', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
|
|
380
|
+
results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'installed', message: 're-pointed stale symlink to current source' });
|
|
381
|
+
} catch (relinkErr: any) {
|
|
382
|
+
// The old link is already gone at this point (unlinkSync above
|
|
383
|
+
// succeeded) — letting this escape to the outer catch below
|
|
384
|
+
// silently fell through to reporting 'already-installed' for a
|
|
385
|
+
// path that no longer existed on disk at all. Windows junctions
|
|
386
|
+
// cannot cross volumes, so a package relocated to a different
|
|
387
|
+
// drive by an update hits exactly this. Mirror the fresh-install
|
|
388
|
+
// path's fallback: try a copy rather than leaving the skill
|
|
389
|
+
// MISSING while claiming success.
|
|
390
|
+
try {
|
|
391
|
+
copyDir(source, absSkillPath);
|
|
392
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'copy', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
|
|
393
|
+
results.push({ tool: id, path: absSkillPath, type: 'copy', status: 'installed', message: `symlink re-point unavailable (${relinkErr?.code ?? relinkErr?.message}); copied instead` });
|
|
394
|
+
} catch (copyErr: any) {
|
|
395
|
+
results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'error', message: `symlink re-point failed (${relinkErr?.code ?? relinkErr?.message}) and the copy fallback also failed: ${copyErr?.message ?? String(copyErr)}` });
|
|
396
|
+
}
|
|
397
|
+
}
|
|
380
398
|
continue;
|
|
381
399
|
}
|
|
382
400
|
}
|
|
383
|
-
} catch { /* fall through to already-installed below */ }
|
|
401
|
+
} catch { /* fall through to already-installed below — lstat/readlink failed (e.g. a TOCTOU race) */ }
|
|
384
402
|
results.push({ tool: id, path: absSkillPath, type, status: 'already-installed' });
|
|
385
|
-
|
|
403
|
+
{
|
|
404
|
+
const isSymlinkNow = isSymlinkPresent(absSkillPath);
|
|
405
|
+
// A symlink always reflects the live package, so re-stamping the
|
|
406
|
+
// current version here is accurate. An owned COPY's on-disk content
|
|
407
|
+
// is NOT touched by this branch — stamping it with the current
|
|
408
|
+
// version anyway would falsely mark it "up to date," permanently
|
|
409
|
+
// defeating reconcileSkillInstalls's staleness refresh (which
|
|
410
|
+
// compares e.version against the current version) on every future
|
|
411
|
+
// run, since this branch runs on every ordinary re-install/re-check.
|
|
412
|
+
// Preserve whatever version (if any) was already recorded instead,
|
|
413
|
+
// so an actually-stale copy still reads as stale until it is really
|
|
414
|
+
// refreshed.
|
|
415
|
+
const existingEntry = manifest.entries.find(e => e.path === absSkillPath);
|
|
416
|
+
const versionField = isSymlinkNow
|
|
417
|
+
? (skillVersion ? { version: skillVersion } : {})
|
|
418
|
+
: (existingEntry?.version ? { version: existingEntry.version } : {});
|
|
419
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkNow ? 'symlink' : 'copy', source, createdAt: now, ...versionField });
|
|
420
|
+
}
|
|
386
421
|
continue;
|
|
387
422
|
}
|
|
388
423
|
if (fs.existsSync(absSkillPath) || isSymlinkPresent(absSkillPath)) {
|
|
@@ -22,6 +22,8 @@ import {
|
|
|
22
22
|
import {
|
|
23
23
|
formatUsersCompact,
|
|
24
24
|
formatSitesCompact,
|
|
25
|
+
formatQuestionsCompact,
|
|
26
|
+
formatAnswersCompact,
|
|
25
27
|
} from './output/compact.ts';
|
|
26
28
|
import type { ExtensionContext, AgentToolResult } from '@earendil-works/pi-coding-agent';
|
|
27
29
|
import { logger } from '../logger.ts';
|
|
@@ -416,6 +418,19 @@ function formatCompact(result: unknown): string {
|
|
|
416
418
|
return formatCompactQuestions(result as Question[]);
|
|
417
419
|
}
|
|
418
420
|
}
|
|
421
|
+
} else if (typeof result === 'object' && result !== null) {
|
|
422
|
+
if ('question' in result && 'answers' in result) {
|
|
423
|
+
// command=get's single-question-with-answers shape — mirrors
|
|
424
|
+
// formatTable's equivalent branch. Without this, format=compact fell
|
|
425
|
+
// through to the raw JSON.stringify fallback below for this command,
|
|
426
|
+
// producing an uncurated dump instead of a compact rendering.
|
|
427
|
+
const r = result as { question: Question; answers: Answer[] };
|
|
428
|
+
let output = formatQuestionsCompact([r.question]);
|
|
429
|
+
if (r.answers.length > 0) {
|
|
430
|
+
output += `\n${formatAnswersCompact(r.answers)}`;
|
|
431
|
+
}
|
|
432
|
+
return output;
|
|
433
|
+
}
|
|
419
434
|
}
|
|
420
435
|
return JSON.stringify(result);
|
|
421
436
|
}
|
|
@@ -8,6 +8,7 @@ import { logger } from '../logger.ts';
|
|
|
8
8
|
import { CircuitBreaker } from '../utils/circuit-breaker.ts';
|
|
9
9
|
import { metrics } from '../utils/metrics.ts';
|
|
10
10
|
import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
|
|
11
|
+
import { createTimeoutSignal } from '../web-research/retry-utils.ts';
|
|
11
12
|
|
|
12
13
|
const API_BASE = 'https://api.stackexchange.com/2.3';
|
|
13
14
|
|
|
@@ -88,32 +89,27 @@ export class StackExchangeClient {
|
|
|
88
89
|
url.searchParams.set('site', 'stackoverflow.com');
|
|
89
90
|
}
|
|
90
91
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
//
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
controller.abort();
|
|
100
|
-
};
|
|
101
|
-
if (signal) {
|
|
102
|
-
signal.addEventListener('abort', abortHandler, { once: true });
|
|
103
|
-
}
|
|
92
|
+
// createTimeoutSignal combines the timeout with an already-provided caller
|
|
93
|
+
// signal via AbortSignal.any, which — unlike a plain `signal.addEventListener
|
|
94
|
+
// ('abort', ...)` — correctly produces an already-aborted combined signal
|
|
95
|
+
// when the caller's signal is aborted BEFORE this call even starts (that
|
|
96
|
+
// 'abort' event already fired; a listener added afterward never sees it, so
|
|
97
|
+
// the request went to the network anyway despite the caller having already
|
|
98
|
+
// cancelled). Matches the same pattern used by the security/*.ts clients.
|
|
99
|
+
const requestSignal = createTimeoutSignal(this._timeout, signal);
|
|
104
100
|
|
|
105
101
|
try {
|
|
106
102
|
const response = await fetch(url.toString(), {
|
|
107
103
|
method: options.method,
|
|
108
|
-
signal:
|
|
104
|
+
signal: requestSignal,
|
|
109
105
|
headers: {
|
|
110
106
|
'Accept': 'application/json',
|
|
111
107
|
},
|
|
112
108
|
});
|
|
113
109
|
|
|
114
|
-
// NB:
|
|
115
|
-
//
|
|
116
|
-
//
|
|
110
|
+
// NB: requestSignal stays armed through the body read below (readJsonCapped) —
|
|
111
|
+
// createTimeoutSignal's underlying timer isn't cleared just because fetch()
|
|
112
|
+
// resolved with headers, so a stalled body still aborts rather than hanging
|
|
117
113
|
// for the whole process lifetime.
|
|
118
114
|
|
|
119
115
|
// The SE API returns a structured JSON wrapper (with error_id) even for
|
|
@@ -193,13 +189,6 @@ export class StackExchangeClient {
|
|
|
193
189
|
}
|
|
194
190
|
|
|
195
191
|
throw error;
|
|
196
|
-
} finally {
|
|
197
|
-
// Single clear point: covers both the success and error paths and keeps
|
|
198
|
-
// the timeout armed across the body read above.
|
|
199
|
-
clearTimeout(timeoutId);
|
|
200
|
-
if (signal) {
|
|
201
|
-
signal.removeEventListener('abort', abortHandler);
|
|
202
|
-
}
|
|
203
192
|
}
|
|
204
193
|
});
|
|
205
194
|
}
|
|
@@ -89,13 +89,3 @@ export function validateAndSanitizeQuery(query: string): string {
|
|
|
89
89
|
|
|
90
90
|
return sanitizeQuery(query);
|
|
91
91
|
}
|
|
92
|
-
|
|
93
|
-
/**
|
|
94
|
-
* Validate complexity level
|
|
95
|
-
*
|
|
96
|
-
* @param complexity - The complexity level to validate
|
|
97
|
-
* @returns True if valid, false otherwise
|
|
98
|
-
*/
|
|
99
|
-
export function validateComplexity(complexity: number): boolean {
|
|
100
|
-
return complexity === 0 || complexity === 1 || complexity === 2 || complexity === 3;
|
|
101
|
-
}
|
package/src/utils/json-utils.ts
CHANGED
|
@@ -356,81 +356,3 @@ export function extractJson<T = unknown>(
|
|
|
356
356
|
error: 'No valid JSON found using any extraction method',
|
|
357
357
|
};
|
|
358
358
|
}
|
|
359
|
-
|
|
360
|
-
/**
|
|
361
|
-
* Result of string array normalization
|
|
362
|
-
*/
|
|
363
|
-
export interface NormalizeArrayResult {
|
|
364
|
-
/** Normalized strings */
|
|
365
|
-
strings: string[];
|
|
366
|
-
/** Count of items that were skipped (empty after normalization) */
|
|
367
|
-
skippedCount: number;
|
|
368
|
-
/** Count of items that were objects and needed extraction */
|
|
369
|
-
extractedCount: number;
|
|
370
|
-
/** Warnings about any issues */
|
|
371
|
-
warnings: string[];
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
/**
|
|
375
|
-
* Normalize string array with detailed feedback
|
|
376
|
-
*
|
|
377
|
-
* Normalizes an array of values to strings with detailed feedback
|
|
378
|
-
* about what was normalized and any issues.
|
|
379
|
-
*
|
|
380
|
-
* @param arr - Array of values to normalize
|
|
381
|
-
* @returns Detailed normalization result
|
|
382
|
-
*/
|
|
383
|
-
export function normalizeStringArrayDetailed(
|
|
384
|
-
arr: unknown[]
|
|
385
|
-
): NormalizeArrayResult {
|
|
386
|
-
const strings: string[] = [];
|
|
387
|
-
const warnings: string[] = [];
|
|
388
|
-
let skippedCount = 0;
|
|
389
|
-
let extractedCount = 0;
|
|
390
|
-
|
|
391
|
-
arr.forEach((item, index) => {
|
|
392
|
-
if (typeof item === 'string') {
|
|
393
|
-
const trimmed = item.trim();
|
|
394
|
-
if (trimmed.length > 0) {
|
|
395
|
-
strings.push(trimmed);
|
|
396
|
-
} else {
|
|
397
|
-
skippedCount++;
|
|
398
|
-
warnings.push(`Item ${index}: empty string`);
|
|
399
|
-
}
|
|
400
|
-
} else if (item !== null && typeof item === 'object') {
|
|
401
|
-
extractedCount++;
|
|
402
|
-
const obj = item as Record<string, unknown>;
|
|
403
|
-
const candidate = obj['query'] ?? obj['topic'] ?? obj['text'] ?? obj['task'];
|
|
404
|
-
|
|
405
|
-
if (typeof candidate === 'string') {
|
|
406
|
-
const trimmed = candidate.trim();
|
|
407
|
-
if (trimmed.length > 0) {
|
|
408
|
-
strings.push(trimmed);
|
|
409
|
-
} else {
|
|
410
|
-
skippedCount++;
|
|
411
|
-
warnings.push(`Item ${index}: object extracted empty string`);
|
|
412
|
-
}
|
|
413
|
-
} else {
|
|
414
|
-
// No recognized key, stringify the whole object
|
|
415
|
-
const stringified = JSON.stringify(item);
|
|
416
|
-
if (stringified.length > 0) {
|
|
417
|
-
strings.push(stringified);
|
|
418
|
-
warnings.push(`Item ${index}: no recognized key, stringified object`);
|
|
419
|
-
} else {
|
|
420
|
-
skippedCount++;
|
|
421
|
-
warnings.push(`Item ${index}: object stringified to empty`);
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
} else {
|
|
425
|
-
const stringified = String(item).trim();
|
|
426
|
-
if (stringified.length > 0) {
|
|
427
|
-
strings.push(stringified);
|
|
428
|
-
} else {
|
|
429
|
-
skippedCount++;
|
|
430
|
-
warnings.push(`Item ${index}: ${typeof item} converted to empty string`);
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
});
|
|
434
|
-
|
|
435
|
-
return { strings, skippedCount, extractedCount, warnings };
|
|
436
|
-
}
|