@lincoln504/pi-research 1.3.3 → 1.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/dist/cli.mjs +242 -95
- package/dist/thread-worker.mjs +122 -50
- package/docs/CHANGELOG.md +76 -0
- package/package.json +1 -1
- package/skills/pi-research/SKILL.md +1 -1
- package/src/core/service-registry.ts +37 -3
- package/src/healthcheck/index.ts +9 -7
- package/src/infrastructure/browser/browser-client.ts +5 -2
- package/src/infrastructure/browser/task-execution-service.ts +21 -5
- package/src/infrastructure/browser/thread-worker-browser.ts +49 -1
- package/src/infrastructure/browser/thread-worker-messaging.ts +70 -25
- package/src/infrastructure/browser/thread-worker.mjs +122 -50
- package/src/infrastructure/knowledge-store-service.ts +27 -2
- package/src/knowledge/chunker.ts +14 -1
- package/src/orchestration/quick-research-orchestrator.ts +13 -2
- package/src/orchestration/research-orchestration-service.ts +16 -3
- package/src/sdk.ts +11 -2
- package/src/security/github-advisories.ts +13 -4
- package/src/security/osv-types.ts +1 -1
- package/src/skill-install/skill-installer.ts +69 -6
- package/src/stackexchange/index.ts +15 -0
- package/src/stackexchange/rest-client.ts +13 -24
- package/src/tools/research-tool-definition.ts +3 -2
- package/src/tools/scrape.ts +6 -4
- package/src/tools/security.ts +13 -4
- package/src/utils/input-validation.ts +0 -10
- package/src/utils/json-utils.ts +0 -78
- package/src/utils/log-rotation.ts +21 -6
- package/src/utils/log-utils.ts +43 -33
- package/src/utils/metrics.ts +0 -8
- package/src/utils/shared-links.ts +0 -14
- package/src/utils/stdio-capture.ts +44 -7
- package/src/utils/text-utils.ts +6 -2
- package/src/utils/url-utils.ts +7 -1
- package/src/web-research/scraper-utils.ts +8 -3
- package/src/web-research/web-scraper.ts +33 -1
- package/src/youtube/potoken.ts +26 -2
- package/src/youtube/transcript-client.ts +5 -2
|
@@ -22,7 +22,7 @@ import { getMaxScrapeBatches, resolveExcludedTools } from '../constants.ts';
|
|
|
22
22
|
import type { ResearchObserver } from '../core/interfaces/observer-interfaces.ts';
|
|
23
23
|
import { HeadlessObserver, makeSafeObserver, type HeadlessObserverOptions } from './headless-observer.ts';
|
|
24
24
|
import { getService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
|
|
25
|
-
import { ServiceNames, type
|
|
25
|
+
import { ServiceNames, type IKnowledgeStoreService, type IResearchSynthesisService, type IResearchOrchestration } from '../core/service-interfaces.ts';
|
|
26
26
|
import type { ResearchSessionService } from './research-session-service.ts';
|
|
27
27
|
import { normalizeUrl, registerScrapedLinks, getCachedScrapedContent } from '../utils/shared-links.ts';
|
|
28
28
|
import { runHealthCheck, isBusyPoolHealthFailure } from '../healthcheck/index.ts';
|
|
@@ -347,7 +347,17 @@ export class QuickResearchOrchestrator {
|
|
|
347
347
|
if (this.config.KNOWLEDGE_STORE_MODE !== 'none') {
|
|
348
348
|
const ksService = await getService<IKnowledgeStoreService>(ServiceNames.KNOWLEDGE_STORE, ctx, container);
|
|
349
349
|
if (ksService.isReady()) {
|
|
350
|
-
|
|
350
|
+
// Resolve directly from ksService rather than the WRITER_QUEUE DI
|
|
351
|
+
// registration — that registration caches the first IWriterQueue it
|
|
352
|
+
// is ever handed forever, so it goes stale after a Knowledge
|
|
353
|
+
// Mode/cwd change disposes and rebuilds ksService's internal writer
|
|
354
|
+
// queue (see research-orchestration-service.ts's storeLinkDescriptions
|
|
355
|
+
// for the full explanation). ksService.getWriterQueue() always
|
|
356
|
+
// reflects what initialize() just settled on.
|
|
357
|
+
const writer = await ksService.getWriterQueue();
|
|
358
|
+
if (!writer) {
|
|
359
|
+
logger.debug('[QuickOrchestrator] Writer queue unavailable, skipping link descriptions');
|
|
360
|
+
} else {
|
|
351
361
|
const citations = parseCitations(result);
|
|
352
362
|
if (citations.length === 0) {
|
|
353
363
|
logger.warn('[QuickOrchestrator] Researcher produced no parseable CITED LINKS — no descriptions stored for this session');
|
|
@@ -375,6 +385,7 @@ export class QuickResearchOrchestrator {
|
|
|
375
385
|
// Drain so concurrent or subsequent sessions see these entries immediately
|
|
376
386
|
// rather than relying solely on shutdownKnowledgeStore's drain.
|
|
377
387
|
if (enqueued > 0) await writer.drain();
|
|
388
|
+
}
|
|
378
389
|
}
|
|
379
390
|
}
|
|
380
391
|
} catch (err) {
|
|
@@ -23,7 +23,6 @@ import { clearSessionCircuitBreaker } from '../infrastructure/browser/browser-er
|
|
|
23
23
|
import { getService, ServiceLifecycle, tryGetService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
|
|
24
24
|
import { ServiceNames } from '../core/service-interfaces.ts';
|
|
25
25
|
import type {
|
|
26
|
-
IWriterQueue,
|
|
27
26
|
IKnowledgeStoreService,
|
|
28
27
|
IHealthRegistryService,
|
|
29
28
|
IResearchOrchestration,
|
|
@@ -548,8 +547,22 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
548
547
|
}
|
|
549
548
|
|
|
550
549
|
const synthesisService = await getService<IResearchSynthesisService>(ServiceNames.RESEARCH_SYNTHESIS_SERVICE, ctx, container);
|
|
551
|
-
|
|
552
|
-
|
|
550
|
+
// Resolve the writer queue directly from the already-fresh ksService rather
|
|
551
|
+
// than through the WRITER_QUEUE DI registration: that registration caches the
|
|
552
|
+
// FIRST IWriterQueue object it is ever handed, forever — WriterQueue's own
|
|
553
|
+
// initialize() is a no-arg lifecycle no-op that never rebuilds its bound
|
|
554
|
+
// store/chunker, so a later Knowledge Mode/cwd change that disposes and
|
|
555
|
+
// rebuilds ksService's internal writer queue left every future getService()
|
|
556
|
+
// call here still returning the stale, disposed one (writes silently dropped
|
|
557
|
+
// via its "store is closing" guard). ksService.getWriterQueue() always
|
|
558
|
+
// reflects whatever initialize() just settled on, which the isReady() check
|
|
559
|
+
// above already confirmed is live.
|
|
560
|
+
const writer = await ksService.getWriterQueue();
|
|
561
|
+
if (!writer) {
|
|
562
|
+
logger.debug('[ResearchOrchestrationService] Writer queue unavailable, skipping link descriptions');
|
|
563
|
+
return;
|
|
564
|
+
}
|
|
565
|
+
|
|
553
566
|
const roundPrefix = `${round}.`;
|
|
554
567
|
let enqueued = 0;
|
|
555
568
|
let researcherCount = 0;
|
package/src/sdk.ts
CHANGED
|
@@ -34,6 +34,7 @@ import { HeadlessObserver } from './orchestration/headless-observer.ts';
|
|
|
34
34
|
import type { Model } from '@earendil-works/pi-ai';
|
|
35
35
|
import { type ExtensionContext, ModelRegistry } from '@earendil-works/pi-coding-agent';
|
|
36
36
|
import { logger, createLogger, setLogger } from './logger.ts';
|
|
37
|
+
import { redactSecrets } from './utils/log-utils.ts';
|
|
37
38
|
import { getConfig, createConfig, validateConfig, resetConfig, type Config } from './config.ts';
|
|
38
39
|
import { metrics, MetricsRegistry, runWithRunRegistry } from './utils/metrics.ts';
|
|
39
40
|
import type { IMetricsSnapshot, RunSummary } from './utils/metrics.ts';
|
|
@@ -633,10 +634,18 @@ export function getLastResearcherOutcome(): ResearcherOutcome | null {
|
|
|
633
634
|
* Emit a single compact line summarizing a run's tracked errors, so they are visible in
|
|
634
635
|
* the operator's log instead of silently accumulating. No-op when the run was clean.
|
|
635
636
|
*/
|
|
636
|
-
function logRunErrorSummary(report: ErrorReport | null, depthLabel: string, status: 'success' | 'error'): void {
|
|
637
|
+
export function logRunErrorSummary(report: ErrorReport | null, depthLabel: string, status: 'success' | 'error'): void {
|
|
637
638
|
if (!report || report.totalErrors === 0) return;
|
|
639
|
+
// Redact BEFORE truncating to 48 chars: p.message is the RAW, unredacted
|
|
640
|
+
// first-seen error message (unlike p.signature, which extractSignature()
|
|
641
|
+
// already strips URLs from) — a scrape/navigation failure routinely echoes
|
|
642
|
+
// the failing URL verbatim, which can carry userinfo credentials
|
|
643
|
+
// (https://user:pass@host/...). A 48-char window makes landing mid-
|
|
644
|
+
// credential likely; truncating first would cut it before redactSecrets'
|
|
645
|
+
// patterns ever see the whole token. Same class of bug as the truncate-
|
|
646
|
+
// before-redact fix in log-utils.ts's redactSecrets itself.
|
|
638
647
|
const top = report.patterns.slice(0, 3)
|
|
639
|
-
.map(p => `"${String(p.message ?? p.signature ?? 'error').slice(0, 48)}" ×${p.count}`)
|
|
648
|
+
.map(p => `"${redactSecrets(String(p.message ?? p.signature ?? 'error')).slice(0, 48)}" ×${p.count}`)
|
|
640
649
|
.join(', ');
|
|
641
650
|
logger.warn(`[SDK] run ${status} with ${report.totalErrors} tracked error(s) across ${report.uniquePatterns} pattern(s) [${depthLabel}]${top ? ` — top: ${top}` : ''}`);
|
|
642
651
|
}
|
|
@@ -102,11 +102,20 @@ export async function searchGitHubAdvisories(
|
|
|
102
102
|
|
|
103
103
|
const [owner, name] = repoParts;
|
|
104
104
|
// Constrain to GitHub's allowed owner/repo character set before interpolating
|
|
105
|
-
// into the API path. encodeURIComponent
|
|
106
|
-
//
|
|
107
|
-
|
|
105
|
+
// into the API path. encodeURIComponent does NOT neutralise traversal here:
|
|
106
|
+
// '.' and '-' are unreserved characters it passes through unescaped, so an
|
|
107
|
+
// owner/name segment of exactly '.' or '..' survives encodeURIComponent
|
|
108
|
+
// intact and is then collapsed by URL/fetch's own path normalization BEFORE
|
|
109
|
+
// the request is sent — e.g. owner=".." turns "/repos/../rate_limit/..."
|
|
110
|
+
// into a request for "/rate_limit/...", escaping the intended
|
|
111
|
+
// /repos/{owner}/{name}/ scope entirely (verified against the real URL
|
|
112
|
+
// parser). The negative lookahead rejects a segment that is ENTIRELY dots
|
|
113
|
+
// (the only shape path normalization treats as a traversal token) while
|
|
114
|
+
// still allowing legitimate names that merely contain dots (node.js,
|
|
115
|
+
// socket.io).
|
|
116
|
+
const repoNameRe = /^(?!\.+$)[A-Za-z0-9._-]+$/;
|
|
108
117
|
if (!repoNameRe.test(owner!) || !repoNameRe.test(name!)) {
|
|
109
|
-
throw new Error(`Invalid repo format: "${options.repo}". Owner and name may only contain letters, digits, '.', '_' and '-'.`);
|
|
118
|
+
throw new Error(`Invalid repo format: "${options.repo}". Owner and name may only contain letters, digits, '.', '_' and '-', and cannot be all dots.`);
|
|
110
119
|
}
|
|
111
120
|
const url = `${GITHUB_API_BASE}/repos/${encodeURIComponent(owner!)}/${encodeURIComponent(name!)}/security-advisories?per_page=${perPage}`;
|
|
112
121
|
|
|
@@ -237,7 +237,7 @@ export function mapOsvItemToVulnerability(item: OsvVulnerability): Vulnerability
|
|
|
237
237
|
for (const cwe of item.database_specific.cwe) {
|
|
238
238
|
if (typeof cwe === 'string') {
|
|
239
239
|
cwes.push(cwe);
|
|
240
|
-
} else if (typeof cwe === 'object' && typeof (cwe as { id?: string }).id === 'string') {
|
|
240
|
+
} else if (typeof cwe === 'object' && cwe !== null && typeof (cwe as { id?: string }).id === 'string') {
|
|
241
241
|
cwes.push((cwe as { id: string }).id);
|
|
242
242
|
}
|
|
243
243
|
}
|
|
@@ -374,15 +374,50 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
|
|
|
374
374
|
continue;
|
|
375
375
|
}
|
|
376
376
|
fs.unlinkSync(absSkillPath);
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
377
|
+
try {
|
|
378
|
+
fs.symlinkSync(source, absSkillPath, process.platform === 'win32' ? 'junction' : 'dir');
|
|
379
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'symlink', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
|
|
380
|
+
results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'installed', message: 're-pointed stale symlink to current source' });
|
|
381
|
+
} catch (relinkErr: any) {
|
|
382
|
+
// The old link is already gone at this point (unlinkSync above
|
|
383
|
+
// succeeded) — letting this escape to the outer catch below
|
|
384
|
+
// silently fell through to reporting 'already-installed' for a
|
|
385
|
+
// path that no longer existed on disk at all. Windows junctions
|
|
386
|
+
// cannot cross volumes, so a package relocated to a different
|
|
387
|
+
// drive by an update hits exactly this. Mirror the fresh-install
|
|
388
|
+
// path's fallback: try a copy rather than leaving the skill
|
|
389
|
+
// MISSING while claiming success.
|
|
390
|
+
try {
|
|
391
|
+
copyDir(source, absSkillPath);
|
|
392
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'copy', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
|
|
393
|
+
results.push({ tool: id, path: absSkillPath, type: 'copy', status: 'installed', message: `symlink re-point unavailable (${relinkErr?.code ?? relinkErr?.message}); copied instead` });
|
|
394
|
+
} catch (copyErr: any) {
|
|
395
|
+
results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'error', message: `symlink re-point failed (${relinkErr?.code ?? relinkErr?.message}) and the copy fallback also failed: ${copyErr?.message ?? String(copyErr)}` });
|
|
396
|
+
}
|
|
397
|
+
}
|
|
380
398
|
continue;
|
|
381
399
|
}
|
|
382
400
|
}
|
|
383
|
-
} catch { /* fall through to already-installed below */ }
|
|
401
|
+
} catch { /* fall through to already-installed below — lstat/readlink failed (e.g. a TOCTOU race) */ }
|
|
384
402
|
results.push({ tool: id, path: absSkillPath, type, status: 'already-installed' });
|
|
385
|
-
|
|
403
|
+
{
|
|
404
|
+
const isSymlinkNow = isSymlinkPresent(absSkillPath);
|
|
405
|
+
// A symlink always reflects the live package, so re-stamping the
|
|
406
|
+
// current version here is accurate. An owned COPY's on-disk content
|
|
407
|
+
// is NOT touched by this branch — stamping it with the current
|
|
408
|
+
// version anyway would falsely mark it "up to date," permanently
|
|
409
|
+
// defeating reconcileSkillInstalls's staleness refresh (which
|
|
410
|
+
// compares e.version against the current version) on every future
|
|
411
|
+
// run, since this branch runs on every ordinary re-install/re-check.
|
|
412
|
+
// Preserve whatever version (if any) was already recorded instead,
|
|
413
|
+
// so an actually-stale copy still reads as stale until it is really
|
|
414
|
+
// refreshed.
|
|
415
|
+
const existingEntry = manifest.entries.find(e => e.path === absSkillPath);
|
|
416
|
+
const versionField = isSymlinkNow
|
|
417
|
+
? (skillVersion ? { version: skillVersion } : {})
|
|
418
|
+
: (existingEntry?.version ? { version: existingEntry.version } : {});
|
|
419
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkNow ? 'symlink' : 'copy', source, createdAt: now, ...versionField });
|
|
420
|
+
}
|
|
386
421
|
continue;
|
|
387
422
|
}
|
|
388
423
|
if (fs.existsSync(absSkillPath) || isSymlinkPresent(absSkillPath)) {
|
|
@@ -612,10 +647,38 @@ export function reconcileSkillInstalls(opts: InstallOptions = {}): ReconcileResu
|
|
|
612
647
|
// Re-point to the current package source (handles update/reinstall).
|
|
613
648
|
try {
|
|
614
649
|
fs.unlinkSync(e.path);
|
|
650
|
+
} catch { keep.push(e); continue; /* old link untouched; retry next startup */ }
|
|
651
|
+
try {
|
|
615
652
|
fs.symlinkSync(source, e.path, process.platform === 'win32' ? 'junction' : 'dir');
|
|
616
653
|
keep.push({ ...e, type: 'symlink', source });
|
|
617
654
|
result.repointed.push(e.path);
|
|
618
|
-
} catch {
|
|
655
|
+
} catch {
|
|
656
|
+
// The old link is already gone at this point (unlinkSync above
|
|
657
|
+
// succeeded). Mirror installSkill()'s fallback: try a copy rather than
|
|
658
|
+
// leaving the skill MISSING on disk while the manifest still claims
|
|
659
|
+
// it's installed (e.g. a Windows cross-volume junction failure).
|
|
660
|
+
// Stage into a dot-prefixed temp dir and rename into place, same as
|
|
661
|
+
// the copy-refresh path above — copying directly into e.path left a
|
|
662
|
+
// partial, SKILL.md-less directory permanently orphaned on disk
|
|
663
|
+
// (unrecognized by isOwnedCopy) when copyDir failed partway (disk
|
|
664
|
+
// full, interrupted process).
|
|
665
|
+
const dir = path.dirname(e.path);
|
|
666
|
+
const base = path.basename(e.path);
|
|
667
|
+
const suffix = crypto.randomBytes(4).toString('hex');
|
|
668
|
+
const fresh = path.join(dir, `.${base}.new-${suffix}`);
|
|
669
|
+
try {
|
|
670
|
+
copyDir(source, fresh);
|
|
671
|
+
fs.renameSync(fresh, e.path);
|
|
672
|
+
keep.push({ ...e, type: 'copy', source });
|
|
673
|
+
result.repointed.push(e.path);
|
|
674
|
+
} catch {
|
|
675
|
+
try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
|
|
676
|
+
// Both the re-point and the copy fallback failed — the skill really
|
|
677
|
+
// is gone from disk now. Drop the entry instead of leaving the
|
|
678
|
+
// manifest claiming a missing skill is still installed.
|
|
679
|
+
result.pruned.push(e.path);
|
|
680
|
+
}
|
|
681
|
+
}
|
|
619
682
|
} else if (!targetLive) {
|
|
620
683
|
// Dangling with no resolvable source (package fully removed): remove the
|
|
621
684
|
// dead link and drop the entry so the agent's skills dir stays clean.
|
|
@@ -22,6 +22,8 @@ import {
|
|
|
22
22
|
import {
|
|
23
23
|
formatUsersCompact,
|
|
24
24
|
formatSitesCompact,
|
|
25
|
+
formatQuestionsCompact,
|
|
26
|
+
formatAnswersCompact,
|
|
25
27
|
} from './output/compact.ts';
|
|
26
28
|
import type { ExtensionContext, AgentToolResult } from '@earendil-works/pi-coding-agent';
|
|
27
29
|
import { logger } from '../logger.ts';
|
|
@@ -416,6 +418,19 @@ function formatCompact(result: unknown): string {
|
|
|
416
418
|
return formatCompactQuestions(result as Question[]);
|
|
417
419
|
}
|
|
418
420
|
}
|
|
421
|
+
} else if (typeof result === 'object' && result !== null) {
|
|
422
|
+
if ('question' in result && 'answers' in result) {
|
|
423
|
+
// command=get's single-question-with-answers shape — mirrors
|
|
424
|
+
// formatTable's equivalent branch. Without this, format=compact fell
|
|
425
|
+
// through to the raw JSON.stringify fallback below for this command,
|
|
426
|
+
// producing an uncurated dump instead of a compact rendering.
|
|
427
|
+
const r = result as { question: Question; answers: Answer[] };
|
|
428
|
+
let output = formatQuestionsCompact([r.question]);
|
|
429
|
+
if (r.answers.length > 0) {
|
|
430
|
+
output += `\n${formatAnswersCompact(r.answers)}`;
|
|
431
|
+
}
|
|
432
|
+
return output;
|
|
433
|
+
}
|
|
419
434
|
}
|
|
420
435
|
return JSON.stringify(result);
|
|
421
436
|
}
|
|
@@ -8,6 +8,7 @@ import { logger } from '../logger.ts';
|
|
|
8
8
|
import { CircuitBreaker } from '../utils/circuit-breaker.ts';
|
|
9
9
|
import { metrics } from '../utils/metrics.ts';
|
|
10
10
|
import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
|
|
11
|
+
import { createTimeoutSignal } from '../web-research/retry-utils.ts';
|
|
11
12
|
|
|
12
13
|
const API_BASE = 'https://api.stackexchange.com/2.3';
|
|
13
14
|
|
|
@@ -88,32 +89,27 @@ export class StackExchangeClient {
|
|
|
88
89
|
url.searchParams.set('site', 'stackoverflow.com');
|
|
89
90
|
}
|
|
90
91
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
//
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
controller.abort();
|
|
100
|
-
};
|
|
101
|
-
if (signal) {
|
|
102
|
-
signal.addEventListener('abort', abortHandler, { once: true });
|
|
103
|
-
}
|
|
92
|
+
// createTimeoutSignal combines the timeout with an already-provided caller
|
|
93
|
+
// signal via AbortSignal.any, which — unlike a plain `signal.addEventListener
|
|
94
|
+
// ('abort', ...)` — correctly produces an already-aborted combined signal
|
|
95
|
+
// when the caller's signal is aborted BEFORE this call even starts (that
|
|
96
|
+
// 'abort' event already fired; a listener added afterward never sees it, so
|
|
97
|
+
// the request went to the network anyway despite the caller having already
|
|
98
|
+
// cancelled). Matches the same pattern used by the security/*.ts clients.
|
|
99
|
+
const requestSignal = createTimeoutSignal(this._timeout, signal);
|
|
104
100
|
|
|
105
101
|
try {
|
|
106
102
|
const response = await fetch(url.toString(), {
|
|
107
103
|
method: options.method,
|
|
108
|
-
signal:
|
|
104
|
+
signal: requestSignal,
|
|
109
105
|
headers: {
|
|
110
106
|
'Accept': 'application/json',
|
|
111
107
|
},
|
|
112
108
|
});
|
|
113
109
|
|
|
114
|
-
// NB:
|
|
115
|
-
//
|
|
116
|
-
//
|
|
110
|
+
// NB: requestSignal stays armed through the body read below (readJsonCapped) —
|
|
111
|
+
// createTimeoutSignal's underlying timer isn't cleared just because fetch()
|
|
112
|
+
// resolved with headers, so a stalled body still aborts rather than hanging
|
|
117
113
|
// for the whole process lifetime.
|
|
118
114
|
|
|
119
115
|
// The SE API returns a structured JSON wrapper (with error_id) even for
|
|
@@ -193,13 +189,6 @@ export class StackExchangeClient {
|
|
|
193
189
|
}
|
|
194
190
|
|
|
195
191
|
throw error;
|
|
196
|
-
} finally {
|
|
197
|
-
// Single clear point: covers both the success and error paths and keeps
|
|
198
|
-
// the timeout armed across the body read above.
|
|
199
|
-
clearTimeout(timeoutId);
|
|
200
|
-
if (signal) {
|
|
201
|
-
signal.removeEventListener('abort', abortHandler);
|
|
202
|
-
}
|
|
203
192
|
}
|
|
204
193
|
});
|
|
205
194
|
}
|
|
@@ -24,6 +24,7 @@ import { ServiceNames } from '../core/service-interfaces.ts';
|
|
|
24
24
|
import type { IResearchOrchestration, IResearchSynthesisService } from '../core/service-interfaces.ts';
|
|
25
25
|
import { metrics, MetricsRegistry, runWithRunRegistry } from '../utils/metrics.ts';
|
|
26
26
|
import { createResearchRunId, logger, createLogger, isVerboseFromEnv, runWithLogger } from '../logger.ts';
|
|
27
|
+
import { redactSecrets } from '../utils/log-utils.ts';
|
|
27
28
|
import { exportResearchReport, appendExportMessage } from '../utils/research-export.ts';
|
|
28
29
|
import { validateAndSanitizeQuery } from '../utils/input-validation.ts';
|
|
29
30
|
import { validateInitialLinks, MAX_INITIAL_LINKS, MAX_INITIAL_LINK_CHARS } from '../utils/url-utils.ts';
|
|
@@ -507,14 +508,14 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
|
|
|
507
508
|
return {
|
|
508
509
|
content: [{
|
|
509
510
|
type: 'text',
|
|
510
|
-
text: `[SYSTEM MESSAGE]: The research operation was halted gracefully because an API rate limit (HTTP 429) was reached. Please inform the user that the operation was stopped due to provider rate limits and they should wait a moment before trying again.\n\nDetails: ${String(error)}`
|
|
511
|
+
text: `[SYSTEM MESSAGE]: The research operation was halted gracefully because an API rate limit (HTTP 429) was reached. Please inform the user that the operation was stopped due to provider rate limits and they should wait a moment before trying again.\n\nDetails: ${redactSecrets(String(error))}`
|
|
511
512
|
}],
|
|
512
513
|
details: {}
|
|
513
514
|
};
|
|
514
515
|
}
|
|
515
516
|
|
|
516
517
|
logger.error('[research] run failed', error);
|
|
517
|
-
return { content: [{ type: 'text', text: `Research failed: ${String(error)}` }], details: {} };
|
|
518
|
+
return { content: [{ type: 'text', text: `Research failed: ${redactSecrets(String(error))}` }], details: {} };
|
|
518
519
|
} finally {
|
|
519
520
|
// Guarantees panel teardown even when the run fails before the inner
|
|
520
521
|
// research loop is reached (e.g. health-check failure), where the inner
|
package/src/tools/scrape.ts
CHANGED
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
MAX_SCRAPE_CONTENT_CHARS_PER_DOC,
|
|
25
25
|
} from '../constants.ts';
|
|
26
26
|
import { truncateWithMarker } from '../utils/text-utils.ts';
|
|
27
|
+
import { redactSecrets } from '../utils/log-utils.ts';
|
|
27
28
|
import { type Config, getConfig } from '../config.ts';
|
|
28
29
|
import { getService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
|
|
29
30
|
import { ServiceNames } from '../core/service-interfaces.ts';
|
|
@@ -261,16 +262,17 @@ export function createScrapeTool(options: {
|
|
|
261
262
|
const concurrency = p.maxConcurrency || defaultConcurrency;
|
|
262
263
|
|
|
263
264
|
try {
|
|
264
|
-
const results = await scrape(urlsToFetch, concurrency, signal,
|
|
265
|
+
const results = await scrape(urlsToFetch, concurrency, signal, config, getGlobalState().researchId, (result) => {
|
|
265
266
|
options.onUrlScrapeResult?.(result.url, result.success);
|
|
266
267
|
}, container);
|
|
267
268
|
freshResults = Array.isArray(results) ? results : [];
|
|
268
269
|
} catch (error) {
|
|
269
|
-
|
|
270
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
271
|
+
logger.error(`[scrape tool] Scrape failed: ${message}`);
|
|
270
272
|
metrics.increment('tool_scrape_calls_total', 1, { status: 'error' });
|
|
271
273
|
return {
|
|
272
|
-
content: [{ type: 'text', text: `# Scrape Failed\n\n${
|
|
273
|
-
details: { error: String(error) },
|
|
274
|
+
content: [{ type: 'text', text: `# Scrape Failed\n\n${redactSecrets(message)}` }],
|
|
275
|
+
details: { error: redactSecrets(String(error)) },
|
|
274
276
|
};
|
|
275
277
|
}
|
|
276
278
|
|
package/src/tools/security.ts
CHANGED
|
@@ -13,6 +13,15 @@ import { getMaxGatheringCalls } from '../constants.ts';
|
|
|
13
13
|
import { metrics } from '../utils/metrics.ts';
|
|
14
14
|
import type { SecuritySearchParams } from '../security/types.ts';
|
|
15
15
|
|
|
16
|
+
const MAX_DESCRIPTION_CHARS = 300;
|
|
17
|
+
|
|
18
|
+
/** Truncates an advisory description for the markdown summary line. */
|
|
19
|
+
function truncateDescription(description: string): string {
|
|
20
|
+
return description.length > MAX_DESCRIPTION_CHARS
|
|
21
|
+
? `${description.substring(0, MAX_DESCRIPTION_CHARS)}...`
|
|
22
|
+
: description;
|
|
23
|
+
}
|
|
24
|
+
|
|
16
25
|
export function createSecuritySearchTool(options: {
|
|
17
26
|
ctx: ExtensionContext;
|
|
18
27
|
tracker: ToolUsageTracker;
|
|
@@ -229,7 +238,7 @@ export function createSecuritySearchTool(options: {
|
|
|
229
238
|
}
|
|
230
239
|
}
|
|
231
240
|
const description = vuln.description;
|
|
232
|
-
markdown += `- **Description:** ${description
|
|
241
|
+
markdown += `- **Description:** ${truncateDescription(description)}\n`;
|
|
233
242
|
if (vuln.knownExploited === true) {
|
|
234
243
|
markdown += '- **[Actively Exploited]**\n';
|
|
235
244
|
}
|
|
@@ -270,7 +279,7 @@ export function createSecuritySearchTool(options: {
|
|
|
270
279
|
markdown += `- **Product:** ${vuln.product}\n`;
|
|
271
280
|
}
|
|
272
281
|
const description = vuln.description;
|
|
273
|
-
markdown += `- **Description:** ${description
|
|
282
|
+
markdown += `- **Description:** ${truncateDescription(description)}\n`;
|
|
274
283
|
if (vuln.dueDate !== undefined) {
|
|
275
284
|
markdown += `- **Due Date:** ${vuln.dueDate}\n`;
|
|
276
285
|
}
|
|
@@ -310,7 +319,7 @@ export function createSecuritySearchTool(options: {
|
|
|
310
319
|
}
|
|
311
320
|
markdown += `- **Summary:** ${adv.summary}\n`;
|
|
312
321
|
const description = adv.description ?? '';
|
|
313
|
-
markdown += `- **Description:** ${description
|
|
322
|
+
markdown += `- **Description:** ${truncateDescription(description)}\n`;
|
|
314
323
|
if (adv.published) {
|
|
315
324
|
markdown += `- **Published:** ${adv.published}\n`;
|
|
316
325
|
}
|
|
@@ -344,7 +353,7 @@ export function createSecuritySearchTool(options: {
|
|
|
344
353
|
markdown += `### ${vuln.id}\n`;
|
|
345
354
|
markdown += `- **Severity:** ${vuln.severity}\n`;
|
|
346
355
|
const description = vuln.description;
|
|
347
|
-
markdown += `- **Description:** ${description
|
|
356
|
+
markdown += `- **Description:** ${truncateDescription(description)}\n`;
|
|
348
357
|
if (vuln.affectedProducts && vuln.affectedProducts.length > 0) {
|
|
349
358
|
markdown += `- **Affected:** ${vuln.affectedProducts.join(', ')}\n`;
|
|
350
359
|
}
|
|
@@ -89,13 +89,3 @@ export function validateAndSanitizeQuery(query: string): string {
|
|
|
89
89
|
|
|
90
90
|
return sanitizeQuery(query);
|
|
91
91
|
}
|
|
92
|
-
|
|
93
|
-
/**
|
|
94
|
-
* Validate complexity level
|
|
95
|
-
*
|
|
96
|
-
* @param complexity - The complexity level to validate
|
|
97
|
-
* @returns True if valid, false otherwise
|
|
98
|
-
*/
|
|
99
|
-
export function validateComplexity(complexity: number): boolean {
|
|
100
|
-
return complexity === 0 || complexity === 1 || complexity === 2 || complexity === 3;
|
|
101
|
-
}
|
package/src/utils/json-utils.ts
CHANGED
|
@@ -356,81 +356,3 @@ export function extractJson<T = unknown>(
|
|
|
356
356
|
error: 'No valid JSON found using any extraction method',
|
|
357
357
|
};
|
|
358
358
|
}
|
|
359
|
-
|
|
360
|
-
/**
|
|
361
|
-
* Result of string array normalization
|
|
362
|
-
*/
|
|
363
|
-
export interface NormalizeArrayResult {
|
|
364
|
-
/** Normalized strings */
|
|
365
|
-
strings: string[];
|
|
366
|
-
/** Count of items that were skipped (empty after normalization) */
|
|
367
|
-
skippedCount: number;
|
|
368
|
-
/** Count of items that were objects and needed extraction */
|
|
369
|
-
extractedCount: number;
|
|
370
|
-
/** Warnings about any issues */
|
|
371
|
-
warnings: string[];
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
/**
|
|
375
|
-
* Normalize string array with detailed feedback
|
|
376
|
-
*
|
|
377
|
-
* Normalizes an array of values to strings with detailed feedback
|
|
378
|
-
* about what was normalized and any issues.
|
|
379
|
-
*
|
|
380
|
-
* @param arr - Array of values to normalize
|
|
381
|
-
* @returns Detailed normalization result
|
|
382
|
-
*/
|
|
383
|
-
export function normalizeStringArrayDetailed(
|
|
384
|
-
arr: unknown[]
|
|
385
|
-
): NormalizeArrayResult {
|
|
386
|
-
const strings: string[] = [];
|
|
387
|
-
const warnings: string[] = [];
|
|
388
|
-
let skippedCount = 0;
|
|
389
|
-
let extractedCount = 0;
|
|
390
|
-
|
|
391
|
-
arr.forEach((item, index) => {
|
|
392
|
-
if (typeof item === 'string') {
|
|
393
|
-
const trimmed = item.trim();
|
|
394
|
-
if (trimmed.length > 0) {
|
|
395
|
-
strings.push(trimmed);
|
|
396
|
-
} else {
|
|
397
|
-
skippedCount++;
|
|
398
|
-
warnings.push(`Item ${index}: empty string`);
|
|
399
|
-
}
|
|
400
|
-
} else if (item !== null && typeof item === 'object') {
|
|
401
|
-
extractedCount++;
|
|
402
|
-
const obj = item as Record<string, unknown>;
|
|
403
|
-
const candidate = obj['query'] ?? obj['topic'] ?? obj['text'] ?? obj['task'];
|
|
404
|
-
|
|
405
|
-
if (typeof candidate === 'string') {
|
|
406
|
-
const trimmed = candidate.trim();
|
|
407
|
-
if (trimmed.length > 0) {
|
|
408
|
-
strings.push(trimmed);
|
|
409
|
-
} else {
|
|
410
|
-
skippedCount++;
|
|
411
|
-
warnings.push(`Item ${index}: object extracted empty string`);
|
|
412
|
-
}
|
|
413
|
-
} else {
|
|
414
|
-
// No recognized key, stringify the whole object
|
|
415
|
-
const stringified = JSON.stringify(item);
|
|
416
|
-
if (stringified.length > 0) {
|
|
417
|
-
strings.push(stringified);
|
|
418
|
-
warnings.push(`Item ${index}: no recognized key, stringified object`);
|
|
419
|
-
} else {
|
|
420
|
-
skippedCount++;
|
|
421
|
-
warnings.push(`Item ${index}: object stringified to empty`);
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
} else {
|
|
425
|
-
const stringified = String(item).trim();
|
|
426
|
-
if (stringified.length > 0) {
|
|
427
|
-
strings.push(stringified);
|
|
428
|
-
} else {
|
|
429
|
-
skippedCount++;
|
|
430
|
-
warnings.push(`Item ${index}: ${typeof item} converted to empty string`);
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
});
|
|
434
|
-
|
|
435
|
-
return { strings, skippedCount, extractedCount, warnings };
|
|
436
|
-
}
|
|
@@ -10,6 +10,25 @@ export interface RotationLogger {
|
|
|
10
10
|
log(...args: unknown[]): void;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
+
// Numeric equivalent of the 'a' flag string (O_WRONLY|O_APPEND|O_CREAT), with
|
|
14
|
+
// O_NOFOLLOW added when the platform defines it (not available on Windows) so
|
|
15
|
+
// the open refuses to follow a symlink an attacker with write access to the
|
|
16
|
+
// log directory could plant at this path in the window between the preceding
|
|
17
|
+
// unlink/rename and this reopen — defense-in-depth on top of an already-narrow
|
|
18
|
+
// threat model (it requires local write access to this directory to begin with).
|
|
19
|
+
const APPEND_CREATE_NOFOLLOW_FLAGS =
|
|
20
|
+
fs.constants.O_WRONLY | fs.constants.O_APPEND | fs.constants.O_CREAT |
|
|
21
|
+
(typeof fs.constants.O_NOFOLLOW === 'number' ? fs.constants.O_NOFOLLOW : 0);
|
|
22
|
+
|
|
23
|
+
/** Best-effort (re)create `filePath` with `mode`, refusing to follow a symlink
|
|
24
|
+
* planted at the path. Failures are swallowed by design — see call sites.
|
|
25
|
+
* Exported for unit testing of the symlink-refusal behavior. */
|
|
26
|
+
export function touchOwnerOnly(filePath: string, mode: number): void {
|
|
27
|
+
try {
|
|
28
|
+
fs.closeSync(fs.openSync(filePath, APPEND_CREATE_NOFOLLOW_FLAGS, mode));
|
|
29
|
+
} catch { /* best-effort — the next append recreates the file */ }
|
|
30
|
+
}
|
|
31
|
+
|
|
13
32
|
export class LogRotation {
|
|
14
33
|
private readonly MAX_LOG_SIZE = 10 * 1024 * 1024; // 10MB max file size
|
|
15
34
|
private readonly MAX_LOG_FILES = 10; // Keep last 10 archived logs
|
|
@@ -30,9 +49,7 @@ export class LogRotation {
|
|
|
30
49
|
// Same rationale as the post-rotation recreate in rotateLogFile: the
|
|
31
50
|
// unlink discarded the 0o600 file, and a umask-default recreate by the
|
|
32
51
|
// next append would drop the owner-only posture.
|
|
33
|
-
|
|
34
|
-
fs.closeSync(fs.openSync(logFile, 'a', 0o600));
|
|
35
|
-
} catch { /* best-effort — the next append recreates the file */ }
|
|
52
|
+
touchOwnerOnly(logFile, 0o600);
|
|
36
53
|
// And, as there, re-stamp the mode: a concurrent append landing between
|
|
37
54
|
// the unlink and the open makes the creation-time 0o600 a no-op.
|
|
38
55
|
try {
|
|
@@ -94,9 +111,7 @@ export class LogRotation {
|
|
|
94
111
|
// world-traversable tmpdir, silently dropping the owner-only posture Logger
|
|
95
112
|
// establishes at construction. Mode applies only at creation, so this is
|
|
96
113
|
// harmless where a file already exists or mode semantics differ (win32).
|
|
97
|
-
|
|
98
|
-
fs.closeSync(fs.openSync(logFile, 'a', 0o600));
|
|
99
|
-
} catch { /* best-effort — the next append recreates the file */ }
|
|
114
|
+
touchOwnerOnly(logFile, 0o600);
|
|
100
115
|
// The mode above applies only if THIS open created the file. Several
|
|
101
116
|
// processes append to this consolidated log, so a concurrent async append
|
|
102
117
|
// can recreate it at umask perms between the rename and the open, turning
|