@lincoln504/pi-research 0.1.13 → 1.0.1

package/src/web-research/retry-utils.ts

@@ -1,3 +1,4 @@
+import { safeUnref } from '../utils/safe-unref.ts';
 /**
  * API Retry and Timeout Utilities
  *
@@ -8,6 +9,7 @@
  */
 
 import { logger } from '../logger.ts';
+import { metrics } from '../utils/metrics.ts';
 
 // ============================================================================
 // Type Definitions
@@ -30,6 +32,34 @@ export interface RetryOptions {
   readonly maxDelay: number;
   readonly label?: string;
   readonly isTransientError?: (error: unknown) => boolean;
+  // When provided, aborting it short-circuits the inter-attempt backoff wait and
+  // stops further retries, so a caller cancel isn't stuck behind a long sleep.
+  readonly signal?: AbortSignal;
+}
+
+/**
+ * Sleep for `ms`, resolving early (rejecting with an AbortError) if `signal`
+ * aborts. The timer is unref'd so it never keeps the event loop alive.
+ */
+export function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(Object.assign(new Error('Aborted'), { name: 'AbortError' }));
+      return;
+    }
+    const timeoutId = setTimeout(() => {
+      if (signal && onAbort) signal.removeEventListener('abort', onAbort);
+      resolve();
+    }, ms);
+    (timeoutId as TimeoutHandle).unref?.();
+    const onAbort = signal
+      ? () => {
+          clearTimeout(timeoutId as unknown as ReturnType<typeof setTimeout>);
+          reject(Object.assign(new Error('Aborted'), { name: 'AbortError' }));
+        }
+      : undefined;
+    if (signal && onAbort) signal.addEventListener('abort', onAbort, { once: true });
+  });
 }
 
 // ============================================================================
@@ -45,25 +75,30 @@ export function createTimeoutSignal(timeoutMs: number, signal?: AbortSignal): Ab
   if (typeof AbortSignal.timeout === 'function') {
     const timeoutSignal = AbortSignal.timeout(timeoutMs);
     if (signal && typeof AbortSignal.any === 'function') {
-      return AbortSignal.any([signal, timeoutSignal]);
+      try {
+        return AbortSignal.any([signal, timeoutSignal]);
+      } catch (e) {
+        // Fallback if AbortSignal.any fails or is not quite right
+        logger.warn(`[createTimeoutSignal] AbortSignal.any failed, falling back to manual combination: ${e}`);
+      }
     }
-    return timeoutSignal;
+    if (!signal) return timeoutSignal;
   }
   
-  // Fallback for older Node.js versions
+  // Fallback for older Node.js versions or if signal combination is needed manually
   const controller = new AbortController();
+  const combinedController = signal ? new AbortController() : controller;
+  
   const timeoutId = setTimeout(() => {
-    controller.abort();
+    combinedController.abort('timeout');
   }, timeoutMs);
   
   // Unref so it doesn't keep the process alive
   if (typeof timeoutId.unref === 'function') {
-    timeoutId.unref();
+    safeUnref(timeoutId);
   }
   
   if (signal) {
-    // Combine signals manually for older versions
-    const combinedController = new AbortController();
     const abortHandler = () => {
       clearTimeout(timeoutId);
       combinedController.abort();
@@ -71,14 +106,19 @@ export function createTimeoutSignal(timeoutMs: number, signal?: AbortSignal): Ab
     
     signal.addEventListener('abort', abortHandler, { once: true });
     
-    return combinedController.signal;
+    // Also clean up signal listener if timeout fires first
+    combinedController.signal.addEventListener('abort', () => {
+      clearTimeout(timeoutId);
+      signal.removeEventListener('abort', abortHandler);
+    }, { once: true });
   }
   
-  return controller.signal;
+  return combinedController.signal;
 }
 
 /**
- * Wraps a promise with a timeout and abort signal support
+ * Wraps a promise with a timeout and abort signal support.
+ * Ensures cleanup of event listeners on resolution/rejection.
  */
 export function withTimeout<T>(
   promise: Promise<T>,
@@ -86,53 +126,53 @@ export function withTimeout<T>(
   label: string,
   signal?: AbortSignal
 ): Promise<T> {
-  const timeoutController = new AbortController();
-  const timeoutSignal = timeoutController.signal;
-  const combinedSignal = signal
-    ? createTimeoutSignal(timeoutMs, signal)
-    : timeoutSignal;
-    
-  logger.log(`[withTimeout] Starting ${label} with timeout ${timeoutMs}ms. External signal aborted: ${signal?.aborted ?? 'no signal'}`);
+  const combinedSignal = createTimeoutSignal(timeoutMs, signal);
 
-  let raceWon = false;
-  let timeoutId: NodeJS.Timeout | undefined;
+  logger.debug(`[withTimeout] Starting ${timeoutMs}ms timeout guard (external signal: ${signal?.aborted ?? 'no signal'})`);
 
-  const wrappedPromise = promise.then((val) => {
-    raceWon = true;
-    if (timeoutId) clearTimeout(timeoutId);
-    return val;
-  });
+  return new Promise<T>((resolve, reject) => {
+    // Early abort check: if signal is already aborted, reject immediately. Already
+    // aborted before we start means an external (user) cancellation, not a timeout —
+    // a clean stop, so log at debug, not error.
+    if (combinedSignal.aborted) {
+      logger.debug(`[withTimeout] ${label} already aborted at start (external cancellation)`);
+      return reject(new Error(`${label} cancelled or timed out`));
+    }
 
-  const rejectPromise = new Promise<T>((_, reject) => {
-    // Timeout handler
-    timeoutId = setTimeout(() => {
-      logger.error(`[withTimeout] ${label} TIMEOUT after ${timeoutMs}ms`);
-      timeoutController.abort();
-      reject(new Error(`${label} timeout after ${timeoutMs}ms`));
-    }, timeoutMs);
+    let settled = false;
+    const abortHandler = () => {
+      if (!settled) {
+        settled = true;
+        // Distinguish a user/external cancellation (clean — debug) from a genuine
+        // timeout (worth a warning). The reject text keeps both words so downstream
+        // transient-classification on "timed out" is unaffected.
+        if (signal?.aborted) {
+          logger.debug(`[withTimeout] ${label} cancelled (external signal)`);
+        } else {
+          logger.warn(`[withTimeout] ${label} timed out after ${timeoutMs}ms`);
+        }
+        reject(new Error(`${label} cancelled or timed out`));
+      }
+    };
 
-    // Unref so it doesn't keep the process alive
-    (timeoutId as TimeoutHandle).unref?.();
+    combinedSignal.addEventListener('abort', abortHandler, { once: true });
 
-    if (combinedSignal.aborted) {
-      logger.error(`[withTimeout] ${label} ALREADY ABORTED at start`);
-      if (timeoutId) clearTimeout(timeoutId);
-      reject(new Error(`${label} cancelled`));
-    } else {
-      combinedSignal.addEventListener('abort', () => {
-        if (!raceWon) {
-          logger.error(`[withTimeout] ${label} ABORT SIGNAL FIRED`);
-          if (timeoutId) clearTimeout(timeoutId);
-          reject(new Error(`${label} cancelled`));
+    promise.then(
+      (val) => {
+        if (!settled) {
+          settled = true;
+          combinedSignal.removeEventListener('abort', abortHandler);
+          resolve(val);
         }
-      }, { once: true });
-    }
-  });
-
-  return Promise.race([wrappedPromise, rejectPromise]).finally(() => {
-    // Clean up on completion or error
-    if (timeoutId) clearTimeout(timeoutId);
-    timeoutController.abort();
+      },
+      (err) => {
+        if (!settled) {
+          settled = true;
+          combinedSignal.removeEventListener('abort', abortHandler);
+          reject(err);
+        }
+      }
+    );
   });
 }
 
@@ -141,34 +181,42 @@ export function withTimeout<T>(
 // ============================================================================
 
 /**
- * Default transient error detector
+ * True if a single error MESSAGE carries a transient signal. Split out so the same
+ * matching is applied both to the top-level error and to each link of a `cause`
+ * chain (undici nests the real socket reason under `error.cause`).
  */
-export function isTransientError(error: unknown): boolean {
-  if (!(error instanceof Error)) {
-    return false;
-  }
-
-  const message = error.message.toLowerCase();
+function messageIsTransient(message: string): boolean {
+  const m = message.toLowerCase();
 
-  // Network errors
-  if (message.includes('econnrefused') || message.includes('enotfound') || message.includes('timeout') || message.includes('etimedout')) {
+  // Network / socket-layer transport failures, including undici's mid-stream aborts:
+  // a dropped streaming response surfaces as `TypeError: terminated` (cause
+  // "other side closed" / UND_ERR_SOCKET / ECONNRESET). "timed out" is NOT included
+  // here on purpose — the generic classifier keeps "timeout"/"etimedout"; the LLM
+  // wrapper's "cancelled or timed out" wording is handled by the planning-service
+  // classifier, so a security/youtube caller's abort string isn't reinterpreted.
+  if (
+    m.includes('econnrefused') || m.includes('enotfound') || m.includes('timeout') ||
+    m.includes('etimedout') || m.includes('econnreset') || m.includes('fetch failed') ||
+    m.includes('terminated') || m.includes('other side closed') ||
+    m.includes('socket hang up') || m.includes('und_err')
+  ) {
     return true;
   }
 
-  // Rate limiting
-  if (message.includes('429') || message.includes('rate') || message.includes('quota')) {
+  // Rate limiting. Anchor on word boundaries so we don't treat "generate",
+  // "moderate" (contain "rate") or "429" embedded in a larger number as
+  // transient — matching the StackExchange client's boundary-anchored guard.
+  if (/\b429\b/.test(m) || /\brate\b/.test(m) || m.includes('quota')) {
     return true;
   }
 
-  // Temporary service unavailability or server errors
+  // Temporary service unavailability, provider overload, or 5xx server errors.
+  // \b5\d\d\b matches a standalone 5xx code without matching a substring of e.g. "50000".
   if (
-    message.includes('503') || 
-    message.includes('500') || 
-    message.includes('502') || 
-    message.includes('504') ||
-    message.includes('temporarily') || 
-    message.includes('unavailable') ||
-    message.includes('http 5')
+    /\b5\d\d\b/.test(m) ||
+    m.includes('temporarily') ||
+    m.includes('unavailable') ||
+    m.includes('overloaded')
   ) {
     return true;
   }
@@ -176,6 +224,34 @@ export function isTransientError(error: unknown): boolean {
   return false;
 }
 
+/**
+ * Default transient error detector. Inspects the error message AND walks the
+ * `cause` chain (bounded) plus any Node/undici error `code`, so a wrapped transport
+ * failure (undici nests the socket reason under `cause`) is still recognized on the
+ * direct-fetch paths where that chain survives.
+ */
+export function isTransientError(error: unknown): boolean {
+  // Reading .message/.cause/.code are plain property accesses on normal Error/undici objects,
+  // but guard against a pathological throwing getter so classification can never crash the
+  // retry loop — fail safe toward "not transient" (i.e. fail fast) if inspection throws.
+  try {
+    let current: unknown = error;
+    for (let depth = 0; current instanceof Error && depth < 5; depth++) {
+      if (messageIsTransient(current.message)) {
+        return true;
+      }
+      const code = (current as { code?: unknown }).code;
+      if (typeof code === 'string' && (/^und_err/i.test(code) || code === 'ECONNRESET' || code === 'ECONNREFUSED' || code === 'ETIMEDOUT' || code === 'ENOTFOUND')) {
+        return true;
+      }
+      current = (current as { cause?: unknown }).cause;
+    }
+  } catch {
+    return false;
+  }
+  return false;
+}
+
 /**
  * Retry helper with exponential backoff and jitter for transient errors.
  *
@@ -188,7 +264,7 @@ export async function retryWithBackoff<T>(
   fn: () => Promise<T>,
   options: Partial<RetryOptions> = {},
 ): Promise<T> {
-  const opts: Required<RetryOptions> = {
+  const opts: Required<Omit<RetryOptions, 'signal'>> & { signal?: AbortSignal } = {
     maxRetries: 3,
     initialDelay: 1000,
     maxDelay: 10000,
@@ -197,16 +273,30 @@ export async function retryWithBackoff<T>(
     ...options,
   };
 
+  const startTime = Date.now();
   let lastError: Error | null = null;
   let attempt = 0;
+  metrics.increment('retry_attempts_total', 1, { label: opts.label });
 
   while (attempt <= opts.maxRetries) {
     try {
-      return await fn();
+      const result = await fn();
+      const duration = Date.now() - startTime;
+      metrics.observe('retry_duration_ms', duration, { label: opts.label, status: 'success' });
+      if (attempt > 0) {
+        metrics.increment('retry_successful_after_retries_total', 1, { attempt: String(attempt), label: opts.label });
+      }
+      return result;
     } catch (error: unknown) {
       lastError = error instanceof Error ? error : new Error(String(error));
 
       if (!opts.isTransientError(lastError) || attempt === opts.maxRetries) {
+        const duration = Date.now() - startTime;
+        metrics.observe('retry_duration_ms', duration, { label: opts.label, status: 'exhausted' });
+        metrics.increment('retry_exhausted_total', 1, { label: opts.label });
+        if (!opts.isTransientError(lastError)) {
+          metrics.increment('retry_non_transient_errors_total', 1, { label: opts.label });
+        }
         throw lastError;
       }
 
@@ -215,13 +305,19 @@ export async function retryWithBackoff<T>(
       const jitter = Math.random() * 0.5 * baseDelay; // ±50% jitter
       const delay = Math.min(baseDelay + jitter, opts.maxDelay);
 
+      metrics.increment('retry_backoff_total', 1, { attempt: String(attempt), label: opts.label });
+      metrics.observe('retry_backoff_delay_ms', delay, { attempt: String(attempt), label: opts.label });
       logger.warn(`[Retry] ${opts.label} failed (attempt ${attempt + 1}/${opts.maxRetries + 1}), retrying in ${Math.round(delay)}ms: ${lastError.message}`);
 
-      // Wait before retrying
-      await new Promise<void>(resolve => {
-        const timeoutId = setTimeout(resolve, delay);
-        (timeoutId as TimeoutHandle).unref?.();
-      });
+      // Wait before retrying — abortable so a caller cancel doesn't sit through
+      // the full backoff. If the wait is aborted, surface the original transient
+      // error (not the synthetic AbortError) so callers see the real cause.
+      try {
+        await abortableDelay(delay, opts.signal);
+      } catch {
+        metrics.increment('retry_aborted_total', 1, { label: opts.label });
+        throw lastError;
+      }
 
       attempt++;
     }

package/src/web-research/scraper-types.ts

@@ -0,0 +1,72 @@
+/**
+ * Scraper Types
+ */
+
+import { USER_AGENTS } from '../utils/user-agent.ts';
+
+export type FetchType = (
+  _input: string | Request,
+  _init?: RequestInit,
+) => Promise<Response>;
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+declare const fetch: FetchType;
+
+// Size limits to prevent OOM attacks
+export const MAX_HTML_SIZE = 25 * 1024 * 1024; // 25MB
+export const MAX_PDF_SIZE = 100 * 1024 * 1024; // 100MB
+
+// SSRF protection patterns
+export const INTERNAL_NETWORK_PATTERNS: ReadonlyArray<RegExp> = [
+  /^127\./,                        // IPv4 loopback
+  /^0\./,                          // IPv4 "this" network
+  /^::1$/,                         // IPv6 loopback
+  /^fe80::/i,                      // IPv6 link-local
+  /^f[cd][0-9a-f]{2}:/i,           // IPv6 unique local — full fc00::/7 block (fc00::–fdff::)
+  /^169\.254\./,                   // IPv4 link-local (and metadata)
+  /^10\./,                         // RFC 1918 Class A private
+  /^192\.168\./,                   // RFC 1918 Class C private
+  /^172\.(1[6-9]|2[0-9]|3[01])\./, // RFC 1918 Class B private
+  /^::ffff:(127\.|0\.|169\.254\.|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)/i, // IPv4-mapped IPv6
+];
+
+// USER_AGENTS re-exported from utils/user-agent.ts for convenience
+export { USER_AGENTS };
+
+export const FILTERED_TAGS = [
+  'nav', 'header', 'footer', 'aside',
+  'script', 'style', 'noscript',
+  'form', 'input', 'select', 'textarea', 'button',
+  'object', 'embed',
+  'svg', 'symbol', 'use', 'defs', 'path', 'circle', 'rect', 'line', 'polygon',
+  'img', 'iframe',
+] as const;
+
+/** Bot detection patterns used by scrapers and browser workers */
+export const BOT_PATTERNS: ReadonlyArray<[string, string]> = [
+  ['_cf_chl_', 'Cloudflare challenge'],
+  ['cf_chl_opt', 'Cloudflare challenge opt'],
+  ['cdn-cgi/challenge-platform', 'Cloudflare challenge platform'],
+  ['ddos-guard', 'DDoS-Guard challenge'],
+  ['Just a moment...', 'Cloudflare interstitial'],
+  ['Checking your browser before accessing', 'Cloudflare interstitial'],
+];
+
+export const IMAGE_LINK_PATTERN = /\[([^\]]*)\]\((data:image\/[^)]+|[^)\s]+\.(?:svg|png|jpe?g|gif|webp|bmp|ico)(?:\?[^)]*)?)\)/gi;
+export const MARKDOWN_IMAGE_PATTERN = /!\[[^\]]*]\((?:data:image\/[^)]+|[^)\s]+)\)/gi;
+
+export type { ScrapeLayerResult } from './types.ts';
+
+export interface NativeHtmlToMarkdownModule {
+  convert(
+    html: string,
+    options?: {
+      headingStyle?: unknown;
+      codeBlockStyle?: unknown;
+      wrap?: boolean;
+    } | null,
+    visitor?: object | null,
+  ): { content?: string };
+  HeadingStyle: { Atx: unknown };
+  CodeBlockStyle: { Backticks: unknown };
+}