@lincoln504/pi-research 1.4.3 → 1.5.0

package/src/web-research/web-scraper.ts

@@ -75,6 +75,15 @@ async function convertToMarkdown(html: string): Promise<string> {
   return converter(html);
 }
 
+// TODO(hardening, medium): this synchronous WASM parse runs on the MAIN
+// process's event loop — a large/complex PDF can block it for seconds,
+// starving every concurrent scrape, the TUI render loop, and timers. The
+// correct fix is a worker_threads-based extraction (mirroring the browser
+// pool's CPU isolation pattern), but it is deliberately NOT done in this
+// pass: pdf-oxide-wasm is dynamically imported here and unit tests mock the
+// module via vitest, which cannot reach into a real worker thread — moving
+// the parse requires a companion test-seam redesign first. Bounded today by
+// MAX_PDF_SIZE (100MB) and the scrape task's own deadline.
 async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
   if (bytes.length > MAX_PDF_SIZE) {
     const sizeMB = Math.round(bytes.length / 1024 / 1024);

package/src/youtube/index.ts

@@ -10,6 +10,8 @@ import type { AgentToolResult } from '@earendil-works/pi-coding-agent';
 import { selectVideoIds } from './video-id.ts';
 import { fetchVideoTranscripts, type VideoTranscript } from './transcript-client.ts';
 import { metrics } from '../utils/metrics.ts';
+import { truncateWithMarker } from '../utils/text-utils.ts';
+import { MAX_SCRAPE_CONTENT_CHARS_PER_DOC } from '../constants.ts';
 
 export interface YoutubeTranscriptCommandOptions {
   urls: string[];
@@ -131,7 +133,15 @@ function formatResults(results: VideoTranscript[], rejected: string[]): string {
     // model can copy verbatim; omitting the line entirely for untitled videos
     // left them without any prompt-visible citation string to reuse.
     md += `**Cite as:** ${r.url}${r.title ? ` — '${r.title}'${r.author ? ` by ${r.author}` : ''}` : ''}\n`;
-    md += `\n${r.text}\n\n---\n\n`;
+    // Per-document cap on content entering LLM context — parity with the
+    // scrape tool's own MAX_SCRAPE_CONTENT_CHARS_PER_DOC guard. The raw HTTP
+    // body is capped at 32MB (readTextCapped) and the knowledge-store
+    // ingestion path truncates too, but this tool's direct response to the
+    // calling agent had no cap of its own: a single long/dense-captioned
+    // video (a multi-hour livestream VOD, worst case with maxVideos up to 5)
+    // could inject an unbounded, multi-megabyte payload straight into the
+    // researcher's context.
+    md += `\n${truncateWithMarker(r.text ?? '', MAX_SCRAPE_CONTENT_CHARS_PER_DOC)}\n\n---\n\n`;
   }
 
   if (failed.length > 0) {