@lincoln504/pi-research 1.3.9 → 1.3.11

package/src/orchestration/research-orchestration-service.ts

@@ -129,6 +129,19 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
       });
       logger.info(`[ResearchOrchestrationService] Acquired research run slot ${runSlot.slotIndex} (cap ${semaphore.getMaxSlots()}).`);
     } catch (err) {
+      const isTerminal =
+        err instanceof ResearchRunCapacityError || (err instanceof Error && err.name === 'AbortError');
+      if (isTerminal) {
+        // The queue notice above may already have announced this run to the
+        // observer (onRunQueued); rethrowing with no terminal event breaks the
+        // "exactly one of onComplete/onError" contract both orchestrators keep —
+        // an observer-driven consumer saw run_queued and then silence. No
+        // orchestrator exists yet, so this cannot double-fire. Observer throws
+        // stay isolated, as everywhere else.
+        try {
+          (observer as { onError?: (e: Error) => void } | undefined)?.onError?.(err as Error);
+        } catch { /* observer isolation */ }
+      }
       if (err instanceof ResearchRunCapacityError) throw err; // capacity exhausted → fail fast
       // A cancel while queueing for a slot must stay cancelled. Fail-open here would
       // start the very run the user just aborted.

package/src/orchestration/research-synthesis-service.ts

@@ -11,7 +11,7 @@
 
 import { parseCitations } from '../utils/text-utils.ts';
 import { normalizeCitations, formatCitedLinks, rewriteCitationMarkers, type GlobalCitation } from '../utils/citation-utils.ts';
-import { lastCitedLinksHeaderIndex } from '../utils/text-utils.ts';
+import { lastCitedLinksHeaderIndex, lastCitedLinksHeader } from '../utils/text-utils.ts';
 import { getScrapedLinks, isTranscribedLink, normalizeUrl } from '../utils/shared-links.ts';
 import { stripTrailingLlmPunctuation } from '../utils/url-utils.ts';
 import { logger } from '../logger.ts';
@@ -321,7 +321,9 @@ export class ResearchSynthesisService implements IService {
           if (globalId !== undefined) localToGlobal.set(localKey, globalId);
           else droppedNumbers.add(localKey);
         });
-        let body = synthesis.slice(0, citedHeaderIdx).trimEnd();
+        // lineStart (not the 'C' offset): slicing at textStart left the tolerated
+        // `##`/`**`/`>` marker dangling as a stray line at the end of the body.
+        let body = synthesis.slice(0, lastCitedLinksHeader(synthesis).lineStart).trimEnd();
         if (localToGlobal.size > 0 || droppedNumbers.size > 0) {
           body = rewriteCitationMarkers(body, localToGlobal, (n) => droppedNumbers.has(n));
         }

package/src/orchestration/researcher-executor.ts

@@ -9,7 +9,7 @@ import type { SystemResearchState } from './deep-research-types.ts';
 import { createResearcherSession } from './researcher.ts';
 import { registerScrapedLinks } from '../utils/shared-links.ts';
 import { ensureAssistantResponse } from '../utils/text-utils.ts';
-import { extractUsage } from '../types/llm.ts';
+import { recordLlmUsage } from '../utils/llm-usage.ts';
 import { logger } from '../logger.ts';
 import { metrics } from '../utils/metrics.ts';
 import { ServiceNames,
@@ -347,13 +347,15 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
 
         const rawUsage = msg['usage'] as any;
         if (rawUsage) {
-          const { tokens, cost } = extractUsage(resolvedModel, rawUsage);
-
+          // Observer exceptions must not kill the researcher, so hand recordLlmUsage
+          // a safeObserve-wrapped sink rather than the raw observer.
+          const { tokens, cost } = recordLlmUsage(resolvedModel, rawUsage, {
+            component: 'researcher',
+            complexity,
+            observer: { onTokensConsumed: (t, c) => safeObserve(() => observer?.onTokensConsumed?.(t, c)) },
+          });
           if (tokens > 0 || cost > 0) {
-            metrics.increment('llm_tokens_total', tokens, { component: 'researcher', complexity: String(complexity) });
-            metrics.increment('llm_cost_total', cost, { component: 'researcher', complexity: String(complexity) });
             safeObserve(() => observer?.onResearcherProgress?.(id, undefined, tokens, cost));
-            safeObserve(() => observer?.onTokensConsumed?.(tokens, cost));
           }
         }
       } else if (event.type === 'tool_execution_start') {

package/src/tools/research-tool-definition.ts

@@ -28,7 +28,7 @@ import { redactSecrets } from '../utils/log-utils.ts';
 import { exportResearchReport, appendExportMessage } from '../utils/research-export.ts';
 import { validateAndSanitizeQuery } from '../utils/input-validation.ts';
 import { validateInitialLinks, MAX_INITIAL_LINKS, MAX_INITIAL_LINK_CHARS } from '../utils/url-utils.ts';
-import { startResearchSession, registerSessionAbort, clearSteeringMessages, getPiActivePanels } from '../orchestration/session-state.ts';
+import { startResearchSession, endResearchSession, registerSessionAbort, clearSteeringMessages, getPiActivePanels } from '../orchestration/session-state.ts';
 import { createResearchTuiManager, hideWorkingIndicator } from '../tui/research-tui-manager.ts';
 import { createCleanupFunction } from '../cleanup/research-cleanup.ts';
 import { createResearchObserver, createObserverState, stopObserverWaveAnimation } from '../observers/research-observer-impl.ts';
@@ -339,6 +339,13 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
             registerSessionAbort(piSessionId, sessionResearchId, internalAbort);
 
             cleanup = async () => {
+              // Free what startResearchSession/registerSessionAbort created — the
+              // failures/failureReasons entries and the AbortController live in the
+              // module-level piSessions map keyed by the STABLE pi session id, so
+              // without this every headless run leaked them for the process
+              // lifetime (the TUI branch frees them via createCleanupFunction, the
+              // SDK in its own finally; this branch freed only steering).
+              endResearchSession(piSessionId, sessionResearchId);
               clearSteeringMessages(piSessionId);
             };
             

package/src/utils/citation-utils.ts

@@ -1,4 +1,4 @@
-import { parseCitations, lastCitedLinksHeaderIndex, type Citation } from './text-utils.ts';
+import { parseCitations, lastCitedLinksHeader, type Citation } from './text-utils.ts';
 import { normalizeUrl } from './shared-links.ts';
 
 export interface GlobalCitation extends Citation {
@@ -66,7 +66,9 @@ export function normalizeCitations(reports: ReadonlyMap<string, string>): {
     // line-leading, all-caps header, never an incidental "cited links" mention in
     // prose (an unanchored split there would drop real findings before the
     // evaluator ever sees them).
-    const headerIdx = lastCitedLinksHeaderIndex(report);
+    // lineStart (not textStart): slicing at the 'C' left the tolerated `##`/`**`/`>`
+    // marker dangling at the end of the kept body.
+    const headerIdx = lastCitedLinksHeader(report).lineStart;
     let content = headerIdx >= 0 ? report.slice(0, headerIdx) : report;
 
     // Renumber local [N] markers to their global ids, and DELETE the ones that could

package/src/utils/text-utils.ts

@@ -253,17 +253,31 @@ function isPlausibleCitationUrl(url: string): boolean {
  * against the `CITED LINKS\n...` block that formatCitedLinks() emits.
  */
 export function lastCitedLinksHeaderIndex(text: string): number {
+  return lastCitedLinksHeader(text).textStart;
+}
+
+/**
+ * Both offsets of the last CITED LINKS header: `textStart` points at the 'C'
+ * (what parseCitations slices FROM), `lineStart` points at the start of the
+ * header line BEFORE any `#`/`>`/`**` markers (what body-replacement slices TO).
+ * Slicing the body at textStart left the tolerated markdown marker dangling at
+ * the end of the body — the rebuilt report then carried a stray `##` or `**`
+ * line whenever the model wrote `## CITED LINKS` instead of the plain form.
+ */
+export function lastCitedLinksHeader(text: string): { lineStart: number; textStart: number } {
   // Case-insensitive but LINE-LEADING: a mid-prose "...the cited links below..." has
   // non-whitespace before it on the line, so it cannot match — only a header at the
   // start of a line (after optional markdown markers) does.
   const re = /(^|\n)([ \t]*(?:#{1,6}[ \t]*|>[ \t]*|\*\*)?)CITED LINKS\b/gi;
-  let last = -1;
+  let lineStart = -1;
+  let textStart = -1;
   let m: RegExpExecArray | null;
   while ((m = re.exec(text)) !== null) {
+    lineStart = m.index + m[1]!.length;
     // Offset of the 'C' in CITED LINKS = match start + leading newline + markers.
-    last = m.index + m[1]!.length + m[2]!.length;
+    textStart = m.index + m[1]!.length + m[2]!.length;
   }
-  return last;
+  return { lineStart, textStart };
 }
 
 /**

package/src/web-research/scraper-utils.ts

@@ -250,27 +250,73 @@ export function ssrfSafeLookup(
   });
 }
 
-// undici Agent used as the per-request dispatcher for scrape fetches. Cached
-// after first construction. `undefined` = not yet attempted, `null` = undici
-// unavailable (fall back to plain fetch with request-time validation only).
-let cachedSsrfDispatcher: unknown | null | undefined = undefined;
+/**
+ * True when the error is one of our SSRF blocks — request-time validation
+ * (message contains "not allowed") or the connect-time DNS pin (`ESSRFBLOCKED`).
+ *
+ * The connect-time error never surfaces directly: undici wraps it in
+ * `TypeError: fetch failed` with the real error down the `cause` chain, so a
+ * message check on the outer error alone files a connect-time block as a generic
+ * fetch error. Walk the chain (bounded — causes can be cyclic) instead.
+ */
+export function isSsrfBlockError(error: unknown): boolean {
+  let e: unknown = error;
+  for (let depth = 0; depth < 5 && e instanceof Error; depth++) {
+    if ((e as NodeJS.ErrnoException).code === 'ESSRFBLOCKED' || e.message.includes('not allowed')) {
+      return true;
+    }
+    e = (e as { cause?: unknown }).cause;
+  }
+  return false;
+}
+
+/** A fetch-shaped callable plus the dispatcher it must be invoked with. */
+export interface SsrfSafeFetcher {
+  /** undici's own fetch — NOT the global one. See getSsrfSafeFetcher. */
+  fetch: (url: string, init: Record<string, unknown>) => Promise<Response>;
+  dispatcher: unknown;
+}
+
+// Cached after first construction. `undefined` = not yet attempted, `null` =
+// undici unavailable (fall back to the global fetch with request-time
+// validation only).
+let cachedSsrfFetcher: SsrfSafeFetcher | null | undefined = undefined;
 
 /**
  * Build (once) an undici Agent whose connector resolves via ssrfSafeLookup, so
  * every scrape fetch — including each manually-followed redirect hop — connects
- * only to a validated public address. Returns null if undici can't be loaded;
- * callers then fall back to plain fetch (request-time validation still applies).
+ * only to a validated public address, PAIRED with the fetch implementation that
+ * can actually accept it.
+ *
+ * The pairing is the whole point. A dispatcher is only honoured by the undici
+ * that created it: Node's global `fetch` is backed by the undici BUILT INTO the
+ * runtime (6.21.0 on Node 22, 7.24.4 on Node 25), while this package depends on
+ * undici 8.x. Handing a v8 `Agent` to the global fetch throws
+ * `UND_ERR_INVALID_ARG: invalid onRequestStart method` on every request, on
+ * every supported Node — which is exactly what it did: the entire fetch scrape
+ * path failed instantly, every URL was retried once and then fell back to the
+ * stealth browser (~4.5s per page against ~250ms), and the connect-time SSRF pin
+ * this dispatcher exists for never engaged even once. The browser fallback made
+ * it invisible: scrapes still succeeded, just 15-20x slower.
+ *
+ * Returning the two together makes the invalid combination unrepresentable —
+ * a caller cannot end up holding a dispatcher without the fetch that honours it.
+ * Returns null if undici can't be loaded; callers then use the global fetch with
+ * no dispatcher (request-time validation still applies).
  */
-export async function getSsrfSafeDispatcher(): Promise<unknown | null> {
-  if (cachedSsrfDispatcher !== undefined) return cachedSsrfDispatcher as unknown | null;
+export async function getSsrfSafeFetcher(): Promise<SsrfSafeFetcher | null> {
+  if (cachedSsrfFetcher !== undefined) return cachedSsrfFetcher;
   try {
-    const { Agent } = await import('undici');
-    cachedSsrfDispatcher = new Agent({ connect: { lookup: ssrfSafeLookup } });
+    const { Agent, fetch: undiciFetch } = await import('undici');
+    cachedSsrfFetcher = {
+      fetch: undiciFetch as unknown as SsrfSafeFetcher['fetch'],
+      dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } }),
+    };
   } catch (e) {
     logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
-    cachedSsrfDispatcher = null;
+    cachedSsrfFetcher = null;
   }
-  return cachedSsrfDispatcher as unknown | null;
+  return cachedSsrfFetcher;
 }
 
 /**

package/src/web-research/web-scraper.ts

@@ -23,7 +23,7 @@ import {
 import {
   FETCH_LAYER_TIMEOUT,
 } from './types.ts';
-import { getRandomUserAgent, extractDomain, validateUrlForSSRF, validateContent, createNativeMarkdownConverter, createJsMarkdownConverter, getSsrfSafeDispatcher, formatErrorWithCause, isBenignScrapeFailure, } from './scraper-utils.ts';
+import { getRandomUserAgent, extractDomain, validateUrlForSSRF, validateContent, createNativeMarkdownConverter, createJsMarkdownConverter, getSsrfSafeFetcher, formatErrorWithCause, isBenignScrapeFailure, isSsrfBlockError, } from './scraper-utils.ts';
 import { isTransientError, abortableDelay } from './retry-utils.ts';
 import { safeUnref } from '../utils/safe-unref.ts';
 import { readBodyCapped, BodyTooLargeError } from '../utils/http-body.ts';
@@ -208,7 +208,7 @@ function decodeHtmlBody(bytes: Uint8Array, contentTypeHeader: string): string {
  * counts are precisely what production incidents are diagnosed from.
  */
 function trackFetchLayerFailure(url: string, error: unknown): void {
-  const isSsrf = error instanceof Error && error.message.includes('not allowed');
+  const isSsrf = isSsrfBlockError(error);
   errorTracker.trackError(error instanceof Error ? error : String(error), {
     component: 'scrapers',
     operation: 'fetch',
@@ -226,7 +226,12 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
   // closing the DNS-rebinding TOCTOU gap between validateUrlForSSRF (request
   // time) and fetch's own re-resolution (connect time). Null when undici is
   // unavailable — request-time validation above still applies.
-  const dispatcher = await getSsrfSafeDispatcher();
+  //
+  // The fetch implementation comes WITH the dispatcher and must be used with it:
+  // a dispatcher is only honoured by the undici that built it, and the global
+  // fetch is backed by Node's bundled undici, not ours. See getSsrfSafeFetcher.
+  const fetcher = await getSsrfSafeFetcher();
+  const doFetch = fetcher?.fetch ?? ((u: string, init: Record<string, unknown>) => fetch(u, init as RequestInit));
 
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), FETCH_LAYER_TIMEOUT);
@@ -257,16 +262,16 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
     const userAgent = getRandomUserAgent();
 
     for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
-      response = await fetch(currentUrl, {
+      response = await doFetch(currentUrl, {
         signal: controller.signal,
         redirect: 'manual',
         headers: {
           'User-Agent': userAgent,
           'Accept': 'text/html,application/xhtml+xml,application/pdf,*/*;q=0.8',
         },
-        // RequestInit's standard type omits undici's `dispatcher`; cast to attach it.
-        ...(dispatcher ? { dispatcher } : {}),
-      } as RequestInit);
+        // Only ever attached alongside the fetch that honours it (see above).
+        ...(fetcher ? { dispatcher: fetcher.dispatcher } : {}),
+      });
 
       if (response.status >= 300 && response.status < 400) {
         const location = response.headers.get('location');
@@ -380,7 +385,7 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
     // call in a retry, and tracking per attempt would record one failure twice —
     // inflating exactly the error counts these numbers are read to diagnose.
     // scrapeSingle calls trackFetchLayerFailure once with the final error.
-    if (error instanceof Error && error.message.includes('not allowed')) {
+    if (isSsrfBlockError(error)) {
       metrics.increment('scrape_operations_total', 1, { layer: 'fetch', status: 'ssrf_blocked' });
       metrics.observe('scrape_latency_ms', Date.now() - fetchStart, { layer: 'fetch', status: 'ssrf_blocked' });
       throw error;