@lincoln504/pi-research 1.3.12 → 1.4.3

package/src/web-research/web-scraper.ts

@@ -123,23 +123,34 @@ async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
 
   try {
     const doc = new WasmPdfDocument(bytes);
-    const pageCount = doc.pageCount();
-    
-    let markdown = `# PDF Document\n\n**Pages:** ${pageCount}\n\n`;
-    
+    // free() is native/WASM-side memory, not GC-reclaimed — it must run on
+    // every exit, including the case toMarkdownAll() AND the per-page fallback
+    // both throw (a malformed/encrypted PDF the codebase already treats as
+    // routine on the open web, so this path fires regularly). It used to sit
+    // after the fallback loop, reachable only when nothing below it threw —
+    // the one case that needed the fallback landing squarely on the one case
+    // that skipped the free(), leaking one WasmPdfDocument per occurrence in a
+    // browser-server process that stays warm across many research sessions.
     try {
-      markdown += doc.toMarkdownAll();
-    } catch {
-      for (let i = 0; i < pageCount; i++) {
-        markdown += `## Page ${i + 1}\n\n${doc.toMarkdown(i)}\n\n`;
+      const pageCount = doc.pageCount();
+
+      let markdown = `# PDF Document\n\n**Pages:** ${pageCount}\n\n`;
+
+      try {
+        markdown += doc.toMarkdownAll();
+      } catch {
+        for (let i = 0; i < pageCount; i++) {
+          markdown += `## Page ${i + 1}\n\n${doc.toMarkdown(i)}\n\n`;
+        }
       }
+
+      const pdfExtractionDuration = Date.now() - pdfExtractionStart;
+      metrics.observe('scrape_pdf_conversion_ms', pdfExtractionDuration);
+      metrics.increment('scrape_pdf_conversions_total', 1, { status: 'success', pages: String(pageCount) });
+      return markdown;
+    } finally {
+      doc.free();
     }
-    
-    doc.free();
-    const pdfExtractionDuration = Date.now() - pdfExtractionStart;
-    metrics.observe('scrape_pdf_conversion_ms', pdfExtractionDuration);
-    metrics.increment('scrape_pdf_conversions_total', 1, { status: 'success', pages: String(pageCount) });
-    return markdown;
   } catch (e) {
     const msg = e instanceof Error ? e.message : String(e);
     // debug, not error: a malformed/encrypted/scanned-image-only PDF is an
@@ -300,7 +311,7 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
     const contentLength = response.headers.get('content-length');
     if (contentLength) {
       const size = parseInt(contentLength, 10);
-      if (contentType.includes('application/pdf') || url.toLowerCase().endsWith('.pdf')) {
+      if (contentType.includes('application/pdf') || currentUrl.toLowerCase().endsWith('.pdf')) {
         if (size > MAX_PDF_SIZE) {
           const sizeMB = Math.round(size / 1024 / 1024);
           logger.warn(`[Scrapers] PDF too large (Content-Length: ${sizeMB}MB, max 100MB), skipping`);
@@ -319,7 +330,7 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
       }
     }
     
-    if (contentType.includes('application/pdf') || url.toLowerCase().endsWith('.pdf')) {
+    if (contentType.includes('application/pdf') || currentUrl.toLowerCase().endsWith('.pdf')) {
       let pdfBytes: Uint8Array;
       try {
         pdfBytes = await readBodyCapped(response, MAX_PDF_SIZE);

package/src/youtube/potoken.ts

@@ -44,6 +44,7 @@
 import { logger } from '../logger.ts';
 import { safeUnref } from '../utils/safe-unref.ts';
 import { metrics } from '../utils/metrics.ts';
+import { readJsonCapped } from '../utils/http-body.ts';
 
 /**
  * Well-known request key for the YouTube web PoToken program. This is a public
@@ -229,7 +230,13 @@ async function doMint(
       body: JSON.stringify([requestKey, botguardResponse]),
       signal: opts.signal,
     });
-    const integrityJson = (await integrityResponse.json()) as unknown[];
+    if (!integrityResponse.ok) {
+      throw new Error(`HTTP ${integrityResponse.status} from attestation server`);
+    }
+    // The attestation response is a short JSON array; readJsonCapped bounds it the
+    // same way as every other externally-controlled body this codebase reads (the
+    // default 32MB cap is already generous for this endpoint).
+    const integrityJson = await readJsonCapped<unknown[]>(integrityResponse);
     const integrityToken = integrityJson?.[0] as string | undefined;
     if (!integrityToken) {
       throw new Error('Integrity token rejected by attestation server (likely a non-residential IP)');

package/src/youtube/transcript-client.ts

@@ -18,6 +18,7 @@
  */
 
 import { withTimeout, retryWithBackoff, createTimeoutSignal } from '../web-research/retry-utils.ts';
+import { getSsrfSafeFetcher } from '../web-research/scraper-utils.ts';
 import { CircuitBreaker } from '../utils/circuit-breaker.ts';
 import { logger } from '../logger.ts';
 import { redactSecrets } from '../utils/log-utils.ts';
@@ -109,7 +110,31 @@ export async function fetchVideoTranscripts(
   } catch {
     // non-fatal: logging stays at default if the API ever changes
   }
-  const fetchImpl: typeof fetch = (input, init) => fetch(input, init);
+  // Connect-time SSRF pin (see scraper-utils.ts's getSsrfSafeFetcher): closes the
+  // gap between assertTrustedTimedtextUrl's host check below and fetch's own
+  // re-resolution at connect time. The host allowlist only constrains WHICH
+  // hostname a redirect can name; it says nothing about what that hostname
+  // resolves to at the moment the socket opens, and this codebase already
+  // treats a resolver answering differently between validation and connect as
+  // a real threat elsewhere (thread-worker-messaging.ts's DNS-rebinding
+  // backstop applies even to already-trusted hosts). One fetchImpl closure
+  // covers every network call in this batch: the BotGuard/attestation fetches
+  // inside mintPoTokens() and the timedtext fetch below. Falls back to the
+  // plain global fetch for non-string/URL input (never seen in practice here)
+  // since the dispatcher pairing requires a URL string; see getSsrfSafeFetcher's
+  // own doc comment for why the fetch and its dispatcher must travel together.
+  const ssrfFetcher = await getSsrfSafeFetcher();
+  const fetchImpl: typeof fetch = ssrfFetcher
+    ? (input, init) => {
+        if (typeof input === 'string' || input instanceof URL) {
+          return ssrfFetcher.fetch(String(input), {
+            ...(init ?? {}),
+            dispatcher: ssrfFetcher.dispatcher,
+          }) as unknown as Promise<Response>;
+        }
+        return fetch(input, init);
+      }
+    : (input, init) => fetch(input, init);
 
   // 1) Throwaway session to obtain stable visitor data for the PoToken binding.
   let visitorData: string;
@@ -245,10 +270,36 @@ async function fetchOne(
           // response headers AND the body read are bounded and tear the socket
           // down on timeout/cancellation (withTimeout alone would not bound text()).
           const fetchSignal = createTimeoutSignal(timeoutMs, signal);
-          const response = await fetchImpl(url, {
-            signal: fetchSignal,
-            headers: { 'User-Agent': 'Mozilla/5.0', 'Accept-Language': lang },
-          });
+          // Manual redirect following, re-validating each hop against the same
+          // trusted-host allowlist as the first hop. `redirect: 'follow' (the
+          // default) would carry the PoToken-bearing request to whatever host a
+          // 3xx Location header names with no revalidation — assertTrustedTimedtextUrl
+          // above only ever checked base_url, the pre-redirect URL. fetchImpl
+          // carries the connect-time IP pin (see its construction above), so each
+          // hop is also protected against a resolver answering differently
+          // between this host check and the moment the socket actually opens.
+          const MAX_REDIRECTS = 5;
+          let currentUrl = url;
+          let response!: Response;
+          for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
+            response = await fetchImpl(currentUrl, {
+              signal: fetchSignal,
+              redirect: 'manual',
+              headers: { 'User-Agent': 'Mozilla/5.0', 'Accept-Language': lang },
+            });
+            if (response.status >= 300 && response.status < 400) {
+              const location = response.headers.get('location');
+              if (!location) break; // No Location header — treat as final response
+              const resolved = new URL(location, currentUrl).href;
+              assertTrustedTimedtextUrl(resolved);
+              // Drain the redirect response before following: an unconsumed body
+              // pins its socket (undici keeps it open until read/cancelled or GC).
+              void response.body?.cancel()?.catch(() => { /* best-effort */ });
+              currentUrl = resolved;
+              continue;
+            }
+            break; // Non-redirect response
+          }
           // Surface the real HTTP status. Without this, a 403 (age-restricted) or
           // 404 (removed) returns HTML that parseJson3 yields '' for, which would
           // be misreported below as a bot-protection/PoToken failure.

package/dist/prompts/system-lead-evaluator.md

@@ -1,118 +0,0 @@
-# Lead Evaluator
-
-You orchestrate the next phase of research.
-
-## Your Context
-- **ROOT QUERY**: {{root_query}}
-- **Current round**: {{round_number}} / {{max_rounds}}
-- **Complexity Level**: {{complexity_label}}
-{{initial_agenda_section}}
-{{previous_queries_section}}
-{{additional_considerations}}
-
-## CRITICAL: Steering Compliance Checkpoint
-- **STRICT STEERING**: If there is an "ADDITIONAL USER GUIDANCE" section above, you MUST treat every point in it as a **mandatory rule and instruction** for your decisions, next round goals, and queries. Do not ignore them. They are directional requirements.
-- **STEERING SATISFACTION**: Before deciding to SYNTHESIZE, you MUST explicitly verify that ALL user steering requirements have been satisfied by the existing findings. If ANY steering requirement remains unaddressed or only superficially covered, you MUST DELEGATE — do NOT synthesize. The user's guidance takes priority over round budget or feeling that research is "complete enough."
-
-{{disabled_tools_section}}
-
----
-
-## Complexity-Aware Decision Thresholds
-
-{{complexity_guidance}}
-
-{{round_phase_guidance}}
-
----
-
-## Decision Framework
-
-**SYNTHESIZE if:** Research meets the complexity-specific synthesis criteria above.
-**DELEGATE if:** Research meets the complexity-specific delegation criteria above. 
-**Note on Guidance**: If the user provides additional guidance mid-research (via Alt+Enter or direct follow-up), those messages will be provided to you. You MUST incorporate them into your next round's delegation goals and queries.
-
-**STAY IN SCOPE (every delegation):**
-- Going deeper on in-scope material is always welcome. Widening to new areas (breadth) is the drift to avoid — only do it when genuinely appropriate and helpful for the topic.
-- A gap is missing depth on what was asked — not a broader topic next to it. Don't add anything the query didn't ask for, even if it came up in earlier findings.
-- Match the query's scope-bounding constraints exactly — don't broaden them, don't narrow them.
-
-**COMMON MISTAKES TO AVOID:**
-- Do NOT synthesize early just because you've done "enough" rounds. Synthesis should only occur when you have comprehensive, high-quality findings across all major topics.
-- Do NOT synthesize with gaps remaining, hoping they can be "filled in later". Complete the research first.
-- Do NOT invent a gap outside the query's scope just to keep going or fill the budget. Finish what was asked; don't add what wasn't.
-
-Use unique, targeted queries for any new researchers.
-
-**Decision**: Return valid JSON in a code block:
-
-**If synthesizing**:
-```json
-{ 
-  "action": "synthesize", 
-  "content": "<Plain prose report with [N] citations, organized by topic. No markdown headings, bullets, or bold text. Followed by the mandatory CITED LINKS section.>" 
-}
-```
-
-**If delegating**:
-```json
-{
-  "action": "delegate",
-  "researchers": [
-    { 
-      "id": "{{round_number}}.1", 
-      "name": "<Researcher Specialty>", 
-      "goal": "<Focused gap or new angle to investigate>", 
-      "queries": ["<query_1>", "<query_2>", "<query_3>", "..."] 
-    }
-  ],
-  "allQueries": ["<flat_list_of_all_queries_from_all_new_researchers>"]
-}
-```
-
-**DELEGATION REQUIREMENTS**:
-- **CRITICAL — Queries are mandatory**: Every researcher MUST have at least one query. Never plan a researcher without queries. Researchers receive ONLY the search results you delegate to them.
-- **Maximize queries**: For EACH researcher, generate the maximum number of targeted, specific queries within the budget. Do not plan fewer than needed — fill the budget. Queries should target primary sources and authoritative evidence.
-- **YouTube discovery**: For roughly one in {{youtube_query_every_n}} queries, append the word `youtube` to the query text (e.g. `"<topic> analysis youtube"`). DuckDuckGo rarely surfaces YouTube on its own, and YouTube links let researchers read video transcripts. Skip only when the gap is clearly unsuited to video.
-- **Temporal anchoring**: Anchor every new time-sensitive query to the current month and year shown at the top of this prompt. Use that exact year (and month where relevant); never default to an older year than the one shown there unless the user explicitly asks about the past.
-- **Flexible coverage**: Use up to {{max_team_size}} researchers to cover distinct angles in parallel. Scale based on research gaps — a single well-targeted researcher is often sufficient for focused gaps.
-- **Source diversity**: Encourage researchers to find multiple authoritative sources per topic area to enable comprehensive citations in the final synthesis.
-
----
-
-## Quality Standards for Delegation
-
-When delegating, ensure:
-- **Query Specificity**: Each new query targets distinct, unexplored territory
-- **No Redundancy**: Do not repeat queries from previous rounds
-- **Specialized Focus**: Each new researcher has a clear, distinct angle
-- **Gap-Driven**: Only delegate when gaps cannot be resolved from existing findings
-- **Progressive Depth**: New queries go deeper, or to a new angle still inside the query's scope — never outside it, and never repeating surface-level coverage
-
----
-
-## Synthesis Protocol (action = synthesize)
-
-1. **Organization**: Organize the report logically **BY TOPIC**. Do NOT structure it by researcher or round.
-2. **Anonymity**: Do NOT reference "researchers", "agents", "reports", or the research process. Present the findings as a direct, unified knowledge base.
-3.  **Master Links List**: A **Global Source List** has been provided to you. Use these sequential numbers [1], [2], [3], etc., for all inline citations. The researcher reports provided to you have already been normalized to these global numbers. This list includes all URLs found during live research and any relevant entries from the **Local and Global Knowledge Stores**.
-4. **Exhaustive Synthesis**: Use ALL findings from ALL reports. Include every fact, date, name, and statistic verbatim. Longer is better.
-5. **Strict Grounding**: Every sentence must come from a report. Use [N] inline citations. No prior knowledge.
-6. **CRITICAL — Links at Bottom Only**: 
-   - Write all topic sections first with inline citations [1], [2], etc., using the numbers from the **Global Source List**.
-   - Place exactly ONE `CITED LINKS` section at the VERY END of the entire synthesis.
-   - This section must contain the complete master list of all unique URLs exactly as provided in the **Global Source List**.
-   - Do NOT include any links within topic sections or subsections.
-   - Format: `[1] https://url.com [Source: ...] — brief description` on each line.
-
-> **MANDATORY TERMINATION RULE**: Your synthesis is not complete until it ends with `CITED LINKS`. Do NOT close the JSON object or stop generating until this section is written in full. A synthesis without `CITED LINKS` is a failed output.
-
----
-
-## Output Requirements
-
-- **Researcher IDs**: Use Round.Index format (e.g. **{{round_number}}.1**, **{{round_number}}.2**).
-- **Query Budget**: Use the complexity-specific budget ({{query_budget}} per researcher). Fill each researcher's query budget completely.
-- **Team Size**: Scale researcher count to match the gaps. Use up to {{max_team_size}} researchers when delegating, but a single well-targeted researcher is often sufficient for focused gaps. Don't pad the team when fewer researchers will cover the remaining gaps efficiently.
-- **Synthesis Quality**: Logical topic-based structure in plain prose, maximal detail, NO mention of researchers, NO markdown headings or bullets.
-- **Format**: ONLY return valid JSON in a code block.

package/src/prompts/system-lead-evaluator.md

@@ -1,118 +0,0 @@
-# Lead Evaluator
-
-You orchestrate the next phase of research.
-
-## Your Context
-- **ROOT QUERY**: {{root_query}}
-- **Current round**: {{round_number}} / {{max_rounds}}
-- **Complexity Level**: {{complexity_label}}
-{{initial_agenda_section}}
-{{previous_queries_section}}
-{{additional_considerations}}
-
-## CRITICAL: Steering Compliance Checkpoint
-- **STRICT STEERING**: If there is an "ADDITIONAL USER GUIDANCE" section above, you MUST treat every point in it as a **mandatory rule and instruction** for your decisions, next round goals, and queries. Do not ignore them. They are directional requirements.
-- **STEERING SATISFACTION**: Before deciding to SYNTHESIZE, you MUST explicitly verify that ALL user steering requirements have been satisfied by the existing findings. If ANY steering requirement remains unaddressed or only superficially covered, you MUST DELEGATE — do NOT synthesize. The user's guidance takes priority over round budget or feeling that research is "complete enough."
-
-{{disabled_tools_section}}
-
----
-
-## Complexity-Aware Decision Thresholds
-
-{{complexity_guidance}}
-
-{{round_phase_guidance}}
-
----
-
-## Decision Framework
-
-**SYNTHESIZE if:** Research meets the complexity-specific synthesis criteria above.
-**DELEGATE if:** Research meets the complexity-specific delegation criteria above. 
-**Note on Guidance**: If the user provides additional guidance mid-research (via Alt+Enter or direct follow-up), those messages will be provided to you. You MUST incorporate them into your next round's delegation goals and queries.
-
-**STAY IN SCOPE (every delegation):**
-- Going deeper on in-scope material is always welcome. Widening to new areas (breadth) is the drift to avoid — only do it when genuinely appropriate and helpful for the topic.
-- A gap is missing depth on what was asked — not a broader topic next to it. Don't add anything the query didn't ask for, even if it came up in earlier findings.
-- Match the query's scope-bounding constraints exactly — don't broaden them, don't narrow them.
-
-**COMMON MISTAKES TO AVOID:**
-- Do NOT synthesize early just because you've done "enough" rounds. Synthesis should only occur when you have comprehensive, high-quality findings across all major topics.
-- Do NOT synthesize with gaps remaining, hoping they can be "filled in later". Complete the research first.
-- Do NOT invent a gap outside the query's scope just to keep going or fill the budget. Finish what was asked; don't add what wasn't.
-
-Use unique, targeted queries for any new researchers.
-
-**Decision**: Return valid JSON in a code block:
-
-**If synthesizing**:
-```json
-{ 
-  "action": "synthesize", 
-  "content": "<Plain prose report with [N] citations, organized by topic. No markdown headings, bullets, or bold text. Followed by the mandatory CITED LINKS section.>" 
-}
-```
-
-**If delegating**:
-```json
-{
-  "action": "delegate",
-  "researchers": [
-    { 
-      "id": "{{round_number}}.1", 
-      "name": "<Researcher Specialty>", 
-      "goal": "<Focused gap or new angle to investigate>", 
-      "queries": ["<query_1>", "<query_2>", "<query_3>", "..."] 
-    }
-  ],
-  "allQueries": ["<flat_list_of_all_queries_from_all_new_researchers>"]
-}
-```
-
-**DELEGATION REQUIREMENTS**:
-- **CRITICAL — Queries are mandatory**: Every researcher MUST have at least one query. Never plan a researcher without queries. Researchers receive ONLY the search results you delegate to them.
-- **Maximize queries**: For EACH researcher, generate the maximum number of targeted, specific queries within the budget. Do not plan fewer than needed — fill the budget. Queries should target primary sources and authoritative evidence.
-- **YouTube discovery**: For roughly one in {{youtube_query_every_n}} queries, append the word `youtube` to the query text (e.g. `"<topic> analysis youtube"`). DuckDuckGo rarely surfaces YouTube on its own, and YouTube links let researchers read video transcripts. Skip only when the gap is clearly unsuited to video.
-- **Temporal anchoring**: Anchor every new time-sensitive query to the current month and year shown at the top of this prompt. Use that exact year (and month where relevant); never default to an older year than the one shown there unless the user explicitly asks about the past.
-- **Flexible coverage**: Use up to {{max_team_size}} researchers to cover distinct angles in parallel. Scale based on research gaps — a single well-targeted researcher is often sufficient for focused gaps.
-- **Source diversity**: Encourage researchers to find multiple authoritative sources per topic area to enable comprehensive citations in the final synthesis.
-
----
-
-## Quality Standards for Delegation
-
-When delegating, ensure:
-- **Query Specificity**: Each new query targets distinct, unexplored territory
-- **No Redundancy**: Do not repeat queries from previous rounds
-- **Specialized Focus**: Each new researcher has a clear, distinct angle
-- **Gap-Driven**: Only delegate when gaps cannot be resolved from existing findings
-- **Progressive Depth**: New queries go deeper, or to a new angle still inside the query's scope — never outside it, and never repeating surface-level coverage
-
----
-
-## Synthesis Protocol (action = synthesize)
-
-1. **Organization**: Organize the report logically **BY TOPIC**. Do NOT structure it by researcher or round.
-2. **Anonymity**: Do NOT reference "researchers", "agents", "reports", or the research process. Present the findings as a direct, unified knowledge base.
-3.  **Master Links List**: A **Global Source List** has been provided to you. Use these sequential numbers [1], [2], [3], etc., for all inline citations. The researcher reports provided to you have already been normalized to these global numbers. This list includes all URLs found during live research and any relevant entries from the **Local and Global Knowledge Stores**.
-4. **Exhaustive Synthesis**: Use ALL findings from ALL reports. Include every fact, date, name, and statistic verbatim. Longer is better.
-5. **Strict Grounding**: Every sentence must come from a report. Use [N] inline citations. No prior knowledge.
-6. **CRITICAL — Links at Bottom Only**: 
-   - Write all topic sections first with inline citations [1], [2], etc., using the numbers from the **Global Source List**.
-   - Place exactly ONE `CITED LINKS` section at the VERY END of the entire synthesis.
-   - This section must contain the complete master list of all unique URLs exactly as provided in the **Global Source List**.
-   - Do NOT include any links within topic sections or subsections.
-   - Format: `[1] https://url.com [Source: ...] — brief description` on each line.
-
-> **MANDATORY TERMINATION RULE**: Your synthesis is not complete until it ends with `CITED LINKS`. Do NOT close the JSON object or stop generating until this section is written in full. A synthesis without `CITED LINKS` is a failed output.
-
----
-
-## Output Requirements
-
-- **Researcher IDs**: Use Round.Index format (e.g. **{{round_number}}.1**, **{{round_number}}.2**).
-- **Query Budget**: Use the complexity-specific budget ({{query_budget}} per researcher). Fill each researcher's query budget completely.
-- **Team Size**: Scale researcher count to match the gaps. Use up to {{max_team_size}} researchers when delegating, but a single well-targeted researcher is often sufficient for focused gaps. Don't pad the team when fewer researchers will cover the remaining gaps efficiently.
-- **Synthesis Quality**: Logical topic-based structure in plain prose, maximal detail, NO mention of researchers, NO markdown headings or bullets.
-- **Format**: ONLY return valid JSON in a code block.