@lincoln504/pi-research 1.3.3 → 1.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +7 -2
  2. package/dist/cli.mjs +242 -95
  3. package/dist/thread-worker.mjs +122 -50
  4. package/docs/CHANGELOG.md +76 -0
  5. package/package.json +1 -1
  6. package/skills/pi-research/SKILL.md +1 -1
  7. package/src/core/service-registry.ts +37 -3
  8. package/src/healthcheck/index.ts +9 -7
  9. package/src/infrastructure/browser/browser-client.ts +5 -2
  10. package/src/infrastructure/browser/task-execution-service.ts +21 -5
  11. package/src/infrastructure/browser/thread-worker-browser.ts +49 -1
  12. package/src/infrastructure/browser/thread-worker-messaging.ts +70 -25
  13. package/src/infrastructure/browser/thread-worker.mjs +122 -50
  14. package/src/infrastructure/knowledge-store-service.ts +27 -2
  15. package/src/knowledge/chunker.ts +14 -1
  16. package/src/orchestration/quick-research-orchestrator.ts +13 -2
  17. package/src/orchestration/research-orchestration-service.ts +16 -3
  18. package/src/sdk.ts +11 -2
  19. package/src/security/github-advisories.ts +13 -4
  20. package/src/security/osv-types.ts +1 -1
  21. package/src/skill-install/skill-installer.ts +69 -6
  22. package/src/stackexchange/index.ts +15 -0
  23. package/src/stackexchange/rest-client.ts +13 -24
  24. package/src/tools/research-tool-definition.ts +3 -2
  25. package/src/tools/scrape.ts +6 -4
  26. package/src/tools/security.ts +13 -4
  27. package/src/utils/input-validation.ts +0 -10
  28. package/src/utils/json-utils.ts +0 -78
  29. package/src/utils/log-rotation.ts +21 -6
  30. package/src/utils/log-utils.ts +43 -33
  31. package/src/utils/metrics.ts +0 -8
  32. package/src/utils/shared-links.ts +0 -14
  33. package/src/utils/stdio-capture.ts +44 -7
  34. package/src/utils/text-utils.ts +6 -2
  35. package/src/utils/url-utils.ts +7 -1
  36. package/src/web-research/scraper-utils.ts +8 -3
  37. package/src/web-research/web-scraper.ts +33 -1
  38. package/src/youtube/potoken.ts +26 -2
  39. package/src/youtube/transcript-client.ts +5 -2
@@ -22,7 +22,7 @@ import { getMaxScrapeBatches, resolveExcludedTools } from '../constants.ts';
22
22
  import type { ResearchObserver } from '../core/interfaces/observer-interfaces.ts';
23
23
  import { HeadlessObserver, makeSafeObserver, type HeadlessObserverOptions } from './headless-observer.ts';
24
24
  import { getService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
25
- import { ServiceNames, type IWriterQueue, type IKnowledgeStoreService, type IResearchSynthesisService, type IResearchOrchestration } from '../core/service-interfaces.ts';
25
+ import { ServiceNames, type IKnowledgeStoreService, type IResearchSynthesisService, type IResearchOrchestration } from '../core/service-interfaces.ts';
26
26
  import type { ResearchSessionService } from './research-session-service.ts';
27
27
  import { normalizeUrl, registerScrapedLinks, getCachedScrapedContent } from '../utils/shared-links.ts';
28
28
  import { runHealthCheck, isBusyPoolHealthFailure } from '../healthcheck/index.ts';
@@ -347,7 +347,17 @@ export class QuickResearchOrchestrator {
347
347
  if (this.config.KNOWLEDGE_STORE_MODE !== 'none') {
348
348
  const ksService = await getService<IKnowledgeStoreService>(ServiceNames.KNOWLEDGE_STORE, ctx, container);
349
349
  if (ksService.isReady()) {
350
- const writer = await getService<IWriterQueue>(ServiceNames.WRITER_QUEUE, ctx, container);
350
+ // Resolve directly from ksService rather than the WRITER_QUEUE DI
351
+ // registration — that registration caches the first IWriterQueue it
352
+ // is ever handed forever, so it goes stale after a Knowledge
353
+ // Mode/cwd change disposes and rebuilds ksService's internal writer
354
+ // queue (see research-orchestration-service.ts's storeLinkDescriptions
355
+ // for the full explanation). ksService.getWriterQueue() always
356
+ // reflects what initialize() just settled on.
357
+ const writer = await ksService.getWriterQueue();
358
+ if (!writer) {
359
+ logger.debug('[QuickOrchestrator] Writer queue unavailable, skipping link descriptions');
360
+ } else {
351
361
  const citations = parseCitations(result);
352
362
  if (citations.length === 0) {
353
363
  logger.warn('[QuickOrchestrator] Researcher produced no parseable CITED LINKS — no descriptions stored for this session');
@@ -375,6 +385,7 @@ export class QuickResearchOrchestrator {
375
385
  // Drain so concurrent or subsequent sessions see these entries immediately
376
386
  // rather than relying solely on shutdownKnowledgeStore's drain.
377
387
  if (enqueued > 0) await writer.drain();
388
+ }
378
389
  }
379
390
  }
380
391
  } catch (err) {
@@ -23,7 +23,6 @@ import { clearSessionCircuitBreaker } from '../infrastructure/browser/browser-er
23
23
  import { getService, ServiceLifecycle, tryGetService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
24
24
  import { ServiceNames } from '../core/service-interfaces.ts';
25
25
  import type {
26
- IWriterQueue,
27
26
  IKnowledgeStoreService,
28
27
  IHealthRegistryService,
29
28
  IResearchOrchestration,
@@ -548,8 +547,22 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
548
547
  }
549
548
 
550
549
  const synthesisService = await getService<IResearchSynthesisService>(ServiceNames.RESEARCH_SYNTHESIS_SERVICE, ctx, container);
551
- const writer = await getService<IWriterQueue>(ServiceNames.WRITER_QUEUE, ctx, container);
552
-
550
+ // Resolve the writer queue directly from the already-fresh ksService rather
551
+ // than through the WRITER_QUEUE DI registration: that registration caches the
552
+ // FIRST IWriterQueue object it is ever handed, forever — WriterQueue's own
553
+ // initialize() is a no-arg lifecycle no-op that never rebuilds its bound
554
+ // store/chunker, so a later Knowledge Mode/cwd change that disposes and
555
+ // rebuilds ksService's internal writer queue left every future getService()
556
+ // call here still returning the stale, disposed one (writes silently dropped
557
+ // via its "store is closing" guard). ksService.getWriterQueue() always
558
+ // reflects whatever initialize() just settled on, which the isReady() check
559
+ // above already confirmed is live.
560
+ const writer = await ksService.getWriterQueue();
561
+ if (!writer) {
562
+ logger.debug('[ResearchOrchestrationService] Writer queue unavailable, skipping link descriptions');
563
+ return;
564
+ }
565
+
553
566
  const roundPrefix = `${round}.`;
554
567
  let enqueued = 0;
555
568
  let researcherCount = 0;
package/src/sdk.ts CHANGED
@@ -34,6 +34,7 @@ import { HeadlessObserver } from './orchestration/headless-observer.ts';
34
34
  import type { Model } from '@earendil-works/pi-ai';
35
35
  import { type ExtensionContext, ModelRegistry } from '@earendil-works/pi-coding-agent';
36
36
  import { logger, createLogger, setLogger } from './logger.ts';
37
+ import { redactSecrets } from './utils/log-utils.ts';
37
38
  import { getConfig, createConfig, validateConfig, resetConfig, type Config } from './config.ts';
38
39
  import { metrics, MetricsRegistry, runWithRunRegistry } from './utils/metrics.ts';
39
40
  import type { IMetricsSnapshot, RunSummary } from './utils/metrics.ts';
@@ -633,10 +634,18 @@ export function getLastResearcherOutcome(): ResearcherOutcome | null {
633
634
  * Emit a single compact line summarizing a run's tracked errors, so they are visible in
634
635
  * the operator's log instead of silently accumulating. No-op when the run was clean.
635
636
  */
636
- function logRunErrorSummary(report: ErrorReport | null, depthLabel: string, status: 'success' | 'error'): void {
637
+ export function logRunErrorSummary(report: ErrorReport | null, depthLabel: string, status: 'success' | 'error'): void {
637
638
  if (!report || report.totalErrors === 0) return;
639
+ // Redact BEFORE truncating to 48 chars: p.message is the RAW, unredacted
640
+ // first-seen error message (unlike p.signature, which extractSignature()
641
+ // already strips URLs from) — a scrape/navigation failure routinely echoes
642
+ // the failing URL verbatim, which can carry userinfo credentials
643
+ // (https://user:pass@host/...). A 48-char window makes landing mid-
644
+ // credential likely; truncating first would cut it before redactSecrets'
645
+ // patterns ever see the whole token. Same class of bug as the truncate-
646
+ // before-redact fix in log-utils.ts's redactSecrets itself.
638
647
  const top = report.patterns.slice(0, 3)
639
- .map(p => `"${String(p.message ?? p.signature ?? 'error').slice(0, 48)}" ×${p.count}`)
648
+ .map(p => `"${redactSecrets(String(p.message ?? p.signature ?? 'error')).slice(0, 48)}" ×${p.count}`)
640
649
  .join(', ');
641
650
  logger.warn(`[SDK] run ${status} with ${report.totalErrors} tracked error(s) across ${report.uniquePatterns} pattern(s) [${depthLabel}]${top ? ` — top: ${top}` : ''}`);
642
651
  }
@@ -102,11 +102,20 @@ export async function searchGitHubAdvisories(
102
102
 
103
103
  const [owner, name] = repoParts;
104
104
  // Constrain to GitHub's allowed owner/repo character set before interpolating
105
- // into the API path. encodeURIComponent already neutralises traversal, but this
106
- // rejects malformed input early instead of issuing a doomed request.
107
- const repoNameRe = /^[A-Za-z0-9._-]+$/;
105
+ // into the API path. encodeURIComponent does NOT neutralise traversal here:
106
+ // '.' and '-' are unreserved characters it passes through unescaped, so an
107
+ // owner/name segment of exactly '.' or '..' survives encodeURIComponent
108
+ // intact and is then collapsed by URL/fetch's own path normalization BEFORE
109
+ // the request is sent — e.g. owner=".." turns "/repos/../rate_limit/..."
110
+ // into a request for "/rate_limit/...", escaping the intended
111
+ // /repos/{owner}/{name}/ scope entirely (verified against the real URL
112
+ // parser). The negative lookahead rejects a segment that is ENTIRELY dots
113
+ // (the only shape path normalization treats as a traversal token) while
114
+ // still allowing legitimate names that merely contain dots (node.js,
115
+ // socket.io).
116
+ const repoNameRe = /^(?!\.+$)[A-Za-z0-9._-]+$/;
108
117
  if (!repoNameRe.test(owner!) || !repoNameRe.test(name!)) {
109
- throw new Error(`Invalid repo format: "${options.repo}". Owner and name may only contain letters, digits, '.', '_' and '-'.`);
118
+ throw new Error(`Invalid repo format: "${options.repo}". Owner and name may only contain letters, digits, '.', '_' and '-', and cannot be all dots.`);
110
119
  }
111
120
  const url = `${GITHUB_API_BASE}/repos/${encodeURIComponent(owner!)}/${encodeURIComponent(name!)}/security-advisories?per_page=${perPage}`;
112
121
 
@@ -237,7 +237,7 @@ export function mapOsvItemToVulnerability(item: OsvVulnerability): Vulnerability
237
237
  for (const cwe of item.database_specific.cwe) {
238
238
  if (typeof cwe === 'string') {
239
239
  cwes.push(cwe);
240
- } else if (typeof cwe === 'object' && typeof (cwe as { id?: string }).id === 'string') {
240
+ } else if (typeof cwe === 'object' && cwe !== null && typeof (cwe as { id?: string }).id === 'string') {
241
241
  cwes.push((cwe as { id: string }).id);
242
242
  }
243
243
  }
@@ -374,15 +374,50 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
374
374
  continue;
375
375
  }
376
376
  fs.unlinkSync(absSkillPath);
377
- fs.symlinkSync(source, absSkillPath, process.platform === 'win32' ? 'junction' : 'dir');
378
- upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'symlink', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
379
- results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'installed', message: 're-pointed stale symlink to current source' });
377
+ try {
378
+ fs.symlinkSync(source, absSkillPath, process.platform === 'win32' ? 'junction' : 'dir');
379
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'symlink', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
380
+ results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'installed', message: 're-pointed stale symlink to current source' });
381
+ } catch (relinkErr: any) {
382
+ // The old link is already gone at this point (unlinkSync above
383
+ // succeeded) — letting this escape to the outer catch below
384
+ // silently fell through to reporting 'already-installed' for a
385
+ // path that no longer existed on disk at all. Windows junctions
386
+ // cannot cross volumes, so a package relocated to a different
387
+ // drive by an update hits exactly this. Mirror the fresh-install
388
+ // path's fallback: try a copy rather than leaving the skill
389
+ // MISSING while claiming success.
390
+ try {
391
+ copyDir(source, absSkillPath);
392
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'copy', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
393
+ results.push({ tool: id, path: absSkillPath, type: 'copy', status: 'installed', message: `symlink re-point unavailable (${relinkErr?.code ?? relinkErr?.message}); copied instead` });
394
+ } catch (copyErr: any) {
395
+ results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'error', message: `symlink re-point failed (${relinkErr?.code ?? relinkErr?.message}) and the copy fallback also failed: ${copyErr?.message ?? String(copyErr)}` });
396
+ }
397
+ }
380
398
  continue;
381
399
  }
382
400
  }
383
- } catch { /* fall through to already-installed below */ }
401
+ } catch { /* fall through to already-installed below — lstat/readlink failed (e.g. a TOCTOU race) */ }
384
402
  results.push({ tool: id, path: absSkillPath, type, status: 'already-installed' });
385
- upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkPresent(absSkillPath) ? 'symlink' : 'copy', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
403
+ {
404
+ const isSymlinkNow = isSymlinkPresent(absSkillPath);
405
+ // A symlink always reflects the live package, so re-stamping the
406
+ // current version here is accurate. An owned COPY's on-disk content
407
+ // is NOT touched by this branch — stamping it with the current
408
+ // version anyway would falsely mark it "up to date," permanently
409
+ // defeating reconcileSkillInstalls's staleness refresh (which
410
+ // compares e.version against the current version) on every future
411
+ // run, since this branch runs on every ordinary re-install/re-check.
412
+ // Preserve whatever version (if any) was already recorded instead,
413
+ // so an actually-stale copy still reads as stale until it is really
414
+ // refreshed.
415
+ const existingEntry = manifest.entries.find(e => e.path === absSkillPath);
416
+ const versionField = isSymlinkNow
417
+ ? (skillVersion ? { version: skillVersion } : {})
418
+ : (existingEntry?.version ? { version: existingEntry.version } : {});
419
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkNow ? 'symlink' : 'copy', source, createdAt: now, ...versionField });
420
+ }
386
421
  continue;
387
422
  }
388
423
  if (fs.existsSync(absSkillPath) || isSymlinkPresent(absSkillPath)) {
@@ -612,10 +647,38 @@ export function reconcileSkillInstalls(opts: InstallOptions = {}): ReconcileResu
612
647
  // Re-point to the current package source (handles update/reinstall).
613
648
  try {
614
649
  fs.unlinkSync(e.path);
650
+ } catch { keep.push(e); continue; /* old link untouched; retry next startup */ }
651
+ try {
615
652
  fs.symlinkSync(source, e.path, process.platform === 'win32' ? 'junction' : 'dir');
616
653
  keep.push({ ...e, type: 'symlink', source });
617
654
  result.repointed.push(e.path);
618
- } catch { keep.push(e); /* leave entry; retry next startup */ }
655
+ } catch {
656
+ // The old link is already gone at this point (unlinkSync above
657
+ // succeeded). Mirror installSkill()'s fallback: try a copy rather than
658
+ // leaving the skill MISSING on disk while the manifest still claims
659
+ // it's installed (e.g. a Windows cross-volume junction failure).
660
+ // Stage into a dot-prefixed temp dir and rename into place, same as
661
+ // the copy-refresh path above — copying directly into e.path left a
662
+ // partial, SKILL.md-less directory permanently orphaned on disk
663
+ // (unrecognized by isOwnedCopy) when copyDir failed partway (disk
664
+ // full, interrupted process).
665
+ const dir = path.dirname(e.path);
666
+ const base = path.basename(e.path);
667
+ const suffix = crypto.randomBytes(4).toString('hex');
668
+ const fresh = path.join(dir, `.${base}.new-${suffix}`);
669
+ try {
670
+ copyDir(source, fresh);
671
+ fs.renameSync(fresh, e.path);
672
+ keep.push({ ...e, type: 'copy', source });
673
+ result.repointed.push(e.path);
674
+ } catch {
675
+ try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
676
+ // Both the re-point and the copy fallback failed — the skill really
677
+ // is gone from disk now. Drop the entry instead of leaving the
678
+ // manifest claiming a missing skill is still installed.
679
+ result.pruned.push(e.path);
680
+ }
681
+ }
619
682
  } else if (!targetLive) {
620
683
  // Dangling with no resolvable source (package fully removed): remove the
621
684
  // dead link and drop the entry so the agent's skills dir stays clean.
@@ -22,6 +22,8 @@ import {
22
22
  import {
23
23
  formatUsersCompact,
24
24
  formatSitesCompact,
25
+ formatQuestionsCompact,
26
+ formatAnswersCompact,
25
27
  } from './output/compact.ts';
26
28
  import type { ExtensionContext, AgentToolResult } from '@earendil-works/pi-coding-agent';
27
29
  import { logger } from '../logger.ts';
@@ -416,6 +418,19 @@ function formatCompact(result: unknown): string {
416
418
  return formatCompactQuestions(result as Question[]);
417
419
  }
418
420
  }
421
+ } else if (typeof result === 'object' && result !== null) {
422
+ if ('question' in result && 'answers' in result) {
423
+ // command=get's single-question-with-answers shape — mirrors
424
+ // formatTable's equivalent branch. Without this, format=compact fell
425
+ // through to the raw JSON.stringify fallback below for this command,
426
+ // producing an uncurated dump instead of a compact rendering.
427
+ const r = result as { question: Question; answers: Answer[] };
428
+ let output = formatQuestionsCompact([r.question]);
429
+ if (r.answers.length > 0) {
430
+ output += `\n${formatAnswersCompact(r.answers)}`;
431
+ }
432
+ return output;
433
+ }
419
434
  }
420
435
  return JSON.stringify(result);
421
436
  }
@@ -8,6 +8,7 @@ import { logger } from '../logger.ts';
8
8
  import { CircuitBreaker } from '../utils/circuit-breaker.ts';
9
9
  import { metrics } from '../utils/metrics.ts';
10
10
  import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
11
+ import { createTimeoutSignal } from '../web-research/retry-utils.ts';
11
12
 
12
13
  const API_BASE = 'https://api.stackexchange.com/2.3';
13
14
 
@@ -88,32 +89,27 @@ export class StackExchangeClient {
88
89
  url.searchParams.set('site', 'stackoverflow.com');
89
90
  }
90
91
 
91
- const controller = new AbortController();
92
- const timeoutId = setTimeout(() => {
93
- controller.abort();
94
- }, this._timeout);
95
-
96
- // Chain the signal if provided
97
- const abortHandler = () => {
98
- clearTimeout(timeoutId);
99
- controller.abort();
100
- };
101
- if (signal) {
102
- signal.addEventListener('abort', abortHandler, { once: true });
103
- }
92
+ // createTimeoutSignal combines the timeout with an already-provided caller
93
+ // signal via AbortSignal.any, which — unlike a plain `signal.addEventListener
94
+ // ('abort', ...)` — correctly produces an already-aborted combined signal
95
+ // when the caller's signal is aborted BEFORE this call even starts (that
96
+ // 'abort' event already fired; a listener added afterward never sees it, so
97
+ // the request went to the network anyway despite the caller having already
98
+ // cancelled). Matches the same pattern used by the security/*.ts clients.
99
+ const requestSignal = createTimeoutSignal(this._timeout, signal);
104
100
 
105
101
  try {
106
102
  const response = await fetch(url.toString(), {
107
103
  method: options.method,
108
- signal: controller.signal,
104
+ signal: requestSignal,
109
105
  headers: {
110
106
  'Accept': 'application/json',
111
107
  },
112
108
  });
113
109
 
114
- // NB: the timeout is cleared in `finally`, not here. Clearing it now —
115
- // when only the headers have arrived — would leave the body read
116
- // (readJsonCapped below) with no timeout, so a stalled body could hang
110
+ // NB: requestSignal stays armed through the body read below (readJsonCapped) —
111
+ // createTimeoutSignal's underlying timer isn't cleared just because fetch()
112
+ // resolved with headers, so a stalled body still aborts rather than hanging
117
113
  // for the whole process lifetime.
118
114
 
119
115
  // The SE API returns a structured JSON wrapper (with error_id) even for
@@ -193,13 +189,6 @@ export class StackExchangeClient {
193
189
  }
194
190
 
195
191
  throw error;
196
- } finally {
197
- // Single clear point: covers both the success and error paths and keeps
198
- // the timeout armed across the body read above.
199
- clearTimeout(timeoutId);
200
- if (signal) {
201
- signal.removeEventListener('abort', abortHandler);
202
- }
203
192
  }
204
193
  });
205
194
  }
@@ -24,6 +24,7 @@ import { ServiceNames } from '../core/service-interfaces.ts';
24
24
  import type { IResearchOrchestration, IResearchSynthesisService } from '../core/service-interfaces.ts';
25
25
  import { metrics, MetricsRegistry, runWithRunRegistry } from '../utils/metrics.ts';
26
26
  import { createResearchRunId, logger, createLogger, isVerboseFromEnv, runWithLogger } from '../logger.ts';
27
+ import { redactSecrets } from '../utils/log-utils.ts';
27
28
  import { exportResearchReport, appendExportMessage } from '../utils/research-export.ts';
28
29
  import { validateAndSanitizeQuery } from '../utils/input-validation.ts';
29
30
  import { validateInitialLinks, MAX_INITIAL_LINKS, MAX_INITIAL_LINK_CHARS } from '../utils/url-utils.ts';
@@ -507,14 +508,14 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
507
508
  return {
508
509
  content: [{
509
510
  type: 'text',
510
- text: `[SYSTEM MESSAGE]: The research operation was halted gracefully because an API rate limit (HTTP 429) was reached. Please inform the user that the operation was stopped due to provider rate limits and they should wait a moment before trying again.\n\nDetails: ${String(error)}`
511
+ text: `[SYSTEM MESSAGE]: The research operation was halted gracefully because an API rate limit (HTTP 429) was reached. Please inform the user that the operation was stopped due to provider rate limits and they should wait a moment before trying again.\n\nDetails: ${redactSecrets(String(error))}`
511
512
  }],
512
513
  details: {}
513
514
  };
514
515
  }
515
516
 
516
517
  logger.error('[research] run failed', error);
517
- return { content: [{ type: 'text', text: `Research failed: ${String(error)}` }], details: {} };
518
+ return { content: [{ type: 'text', text: `Research failed: ${redactSecrets(String(error))}` }], details: {} };
518
519
  } finally {
519
520
  // Guarantees panel teardown even when the run fails before the inner
520
521
  // research loop is reached (e.g. health-check failure), where the inner
@@ -24,6 +24,7 @@ import {
24
24
  MAX_SCRAPE_CONTENT_CHARS_PER_DOC,
25
25
  } from '../constants.ts';
26
26
  import { truncateWithMarker } from '../utils/text-utils.ts';
27
+ import { redactSecrets } from '../utils/log-utils.ts';
27
28
  import { type Config, getConfig } from '../config.ts';
28
29
  import { getService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
29
30
  import { ServiceNames } from '../core/service-interfaces.ts';
@@ -261,16 +262,17 @@ export function createScrapeTool(options: {
261
262
  const concurrency = p.maxConcurrency || defaultConcurrency;
262
263
 
263
264
  try {
264
- const results = await scrape(urlsToFetch, concurrency, signal, options.config, getGlobalState().researchId, (result) => {
265
+ const results = await scrape(urlsToFetch, concurrency, signal, config, getGlobalState().researchId, (result) => {
265
266
  options.onUrlScrapeResult?.(result.url, result.success);
266
267
  }, container);
267
268
  freshResults = Array.isArray(results) ? results : [];
268
269
  } catch (error) {
269
- logger.error(`[scrape tool] Scrape failed: ${error instanceof Error ? error.message : String(error)}`);
270
+ const message = error instanceof Error ? error.message : String(error);
271
+ logger.error(`[scrape tool] Scrape failed: ${message}`);
270
272
  metrics.increment('tool_scrape_calls_total', 1, { status: 'error' });
271
273
  return {
272
- content: [{ type: 'text', text: `# Scrape Failed\n\n${error instanceof Error ? error.message : String(error)}` }],
273
- details: { error: String(error) },
274
+ content: [{ type: 'text', text: `# Scrape Failed\n\n${redactSecrets(message)}` }],
275
+ details: { error: redactSecrets(String(error)) },
274
276
  };
275
277
  }
276
278
 
@@ -13,6 +13,15 @@ import { getMaxGatheringCalls } from '../constants.ts';
13
13
  import { metrics } from '../utils/metrics.ts';
14
14
  import type { SecuritySearchParams } from '../security/types.ts';
15
15
 
16
+ const MAX_DESCRIPTION_CHARS = 300;
17
+
18
+ /** Truncates an advisory description for the markdown summary line. */
19
+ function truncateDescription(description: string): string {
20
+ return description.length > MAX_DESCRIPTION_CHARS
21
+ ? `${description.substring(0, MAX_DESCRIPTION_CHARS)}...`
22
+ : description;
23
+ }
24
+
16
25
  export function createSecuritySearchTool(options: {
17
26
  ctx: ExtensionContext;
18
27
  tracker: ToolUsageTracker;
@@ -229,7 +238,7 @@ export function createSecuritySearchTool(options: {
229
238
  }
230
239
  }
231
240
  const description = vuln.description;
232
- markdown += `- **Description:** ${description.length > 300 ? `${description.substring(0, 300)}...` : description}\n`;
241
+ markdown += `- **Description:** ${truncateDescription(description)}\n`;
233
242
  if (vuln.knownExploited === true) {
234
243
  markdown += '- **[Actively Exploited]**\n';
235
244
  }
@@ -270,7 +279,7 @@ export function createSecuritySearchTool(options: {
270
279
  markdown += `- **Product:** ${vuln.product}\n`;
271
280
  }
272
281
  const description = vuln.description;
273
- markdown += `- **Description:** ${description.length > 300 ? `${description.substring(0, 300)}...` : description}\n`;
282
+ markdown += `- **Description:** ${truncateDescription(description)}\n`;
274
283
  if (vuln.dueDate !== undefined) {
275
284
  markdown += `- **Due Date:** ${vuln.dueDate}\n`;
276
285
  }
@@ -310,7 +319,7 @@ export function createSecuritySearchTool(options: {
310
319
  }
311
320
  markdown += `- **Summary:** ${adv.summary}\n`;
312
321
  const description = adv.description ?? '';
313
- markdown += `- **Description:** ${description.length > 300 ? `${description.substring(0, 300)}...` : description}\n`;
322
+ markdown += `- **Description:** ${truncateDescription(description)}\n`;
314
323
  if (adv.published) {
315
324
  markdown += `- **Published:** ${adv.published}\n`;
316
325
  }
@@ -344,7 +353,7 @@ export function createSecuritySearchTool(options: {
344
353
  markdown += `### ${vuln.id}\n`;
345
354
  markdown += `- **Severity:** ${vuln.severity}\n`;
346
355
  const description = vuln.description;
347
- markdown += `- **Description:** ${description.length > 300 ? `${description.substring(0, 300)}...` : description}\n`;
356
+ markdown += `- **Description:** ${truncateDescription(description)}\n`;
348
357
  if (vuln.affectedProducts && vuln.affectedProducts.length > 0) {
349
358
  markdown += `- **Affected:** ${vuln.affectedProducts.join(', ')}\n`;
350
359
  }
@@ -89,13 +89,3 @@ export function validateAndSanitizeQuery(query: string): string {
89
89
 
90
90
  return sanitizeQuery(query);
91
91
  }
92
-
93
- /**
94
- * Validate complexity level
95
- *
96
- * @param complexity - The complexity level to validate
97
- * @returns True if valid, false otherwise
98
- */
99
- export function validateComplexity(complexity: number): boolean {
100
- return complexity === 0 || complexity === 1 || complexity === 2 || complexity === 3;
101
- }
@@ -356,81 +356,3 @@ export function extractJson<T = unknown>(
356
356
  error: 'No valid JSON found using any extraction method',
357
357
  };
358
358
  }
359
-
360
- /**
361
- * Result of string array normalization
362
- */
363
- export interface NormalizeArrayResult {
364
- /** Normalized strings */
365
- strings: string[];
366
- /** Count of items that were skipped (empty after normalization) */
367
- skippedCount: number;
368
- /** Count of items that were objects and needed extraction */
369
- extractedCount: number;
370
- /** Warnings about any issues */
371
- warnings: string[];
372
- }
373
-
374
- /**
375
- * Normalize string array with detailed feedback
376
- *
377
- * Normalizes an array of values to strings with detailed feedback
378
- * about what was normalized and any issues.
379
- *
380
- * @param arr - Array of values to normalize
381
- * @returns Detailed normalization result
382
- */
383
- export function normalizeStringArrayDetailed(
384
- arr: unknown[]
385
- ): NormalizeArrayResult {
386
- const strings: string[] = [];
387
- const warnings: string[] = [];
388
- let skippedCount = 0;
389
- let extractedCount = 0;
390
-
391
- arr.forEach((item, index) => {
392
- if (typeof item === 'string') {
393
- const trimmed = item.trim();
394
- if (trimmed.length > 0) {
395
- strings.push(trimmed);
396
- } else {
397
- skippedCount++;
398
- warnings.push(`Item ${index}: empty string`);
399
- }
400
- } else if (item !== null && typeof item === 'object') {
401
- extractedCount++;
402
- const obj = item as Record<string, unknown>;
403
- const candidate = obj['query'] ?? obj['topic'] ?? obj['text'] ?? obj['task'];
404
-
405
- if (typeof candidate === 'string') {
406
- const trimmed = candidate.trim();
407
- if (trimmed.length > 0) {
408
- strings.push(trimmed);
409
- } else {
410
- skippedCount++;
411
- warnings.push(`Item ${index}: object extracted empty string`);
412
- }
413
- } else {
414
- // No recognized key, stringify the whole object
415
- const stringified = JSON.stringify(item);
416
- if (stringified.length > 0) {
417
- strings.push(stringified);
418
- warnings.push(`Item ${index}: no recognized key, stringified object`);
419
- } else {
420
- skippedCount++;
421
- warnings.push(`Item ${index}: object stringified to empty`);
422
- }
423
- }
424
- } else {
425
- const stringified = String(item).trim();
426
- if (stringified.length > 0) {
427
- strings.push(stringified);
428
- } else {
429
- skippedCount++;
430
- warnings.push(`Item ${index}: ${typeof item} converted to empty string`);
431
- }
432
- }
433
- });
434
-
435
- return { strings, skippedCount, extractedCount, warnings };
436
- }
@@ -10,6 +10,25 @@ export interface RotationLogger {
10
10
  log(...args: unknown[]): void;
11
11
  }
12
12
 
13
+ // Numeric equivalent of the 'a' flag string (O_WRONLY|O_APPEND|O_CREAT), with
14
+ // O_NOFOLLOW added when the platform defines it (not available on Windows) so
15
+ // the open refuses to follow a symlink an attacker with write access to the
16
+ // log directory could plant at this path in the window between the preceding
17
+ // unlink/rename and this reopen — defense-in-depth on top of an already-narrow
18
+ // threat model (it requires local write access to this directory to begin with).
19
+ const APPEND_CREATE_NOFOLLOW_FLAGS =
20
+ fs.constants.O_WRONLY | fs.constants.O_APPEND | fs.constants.O_CREAT |
21
+ (typeof fs.constants.O_NOFOLLOW === 'number' ? fs.constants.O_NOFOLLOW : 0);
22
+
23
+ /** Best-effort (re)create `filePath` with `mode`, refusing to follow a symlink
24
+ * planted at the path. Failures are swallowed by design — see call sites.
25
+ * Exported for unit testing of the symlink-refusal behavior. */
26
+ export function touchOwnerOnly(filePath: string, mode: number): void {
27
+ try {
28
+ fs.closeSync(fs.openSync(filePath, APPEND_CREATE_NOFOLLOW_FLAGS, mode));
29
+ } catch { /* best-effort — the next append recreates the file */ }
30
+ }
31
+
13
32
  export class LogRotation {
14
33
  private readonly MAX_LOG_SIZE = 10 * 1024 * 1024; // 10MB max file size
15
34
  private readonly MAX_LOG_FILES = 10; // Keep last 10 archived logs
@@ -30,9 +49,7 @@ export class LogRotation {
30
49
  // Same rationale as the post-rotation recreate in rotateLogFile: the
31
50
  // unlink discarded the 0o600 file, and a umask-default recreate by the
32
51
  // next append would drop the owner-only posture.
33
- try {
34
- fs.closeSync(fs.openSync(logFile, 'a', 0o600));
35
- } catch { /* best-effort — the next append recreates the file */ }
52
+ touchOwnerOnly(logFile, 0o600);
36
53
  // And, as there, re-stamp the mode: a concurrent append landing between
37
54
  // the unlink and the open makes the creation-time 0o600 a no-op.
38
55
  try {
@@ -94,9 +111,7 @@ export class LogRotation {
94
111
  // world-traversable tmpdir, silently dropping the owner-only posture Logger
95
112
  // establishes at construction. Mode applies only at creation, so this is
96
113
  // harmless where a file already exists or mode semantics differ (win32).
97
- try {
98
- fs.closeSync(fs.openSync(logFile, 'a', 0o600));
99
- } catch { /* best-effort — the next append recreates the file */ }
114
+ touchOwnerOnly(logFile, 0o600);
100
115
  // The mode above applies only if THIS open created the file. Several
101
116
  // processes append to this consolidated log, so a concurrent async append
102
117
  // can recreate it at umask perms between the rename and the open, turning