@lincoln504/pi-research 1.3.2 → 1.3.4

package/src/utils/input-validation.ts

@@ -89,13 +89,3 @@ export function validateAndSanitizeQuery(query: string): string {
 
   return sanitizeQuery(query);
 }
-
-/**
- * Validate complexity level
- *
- * @param complexity - The complexity level to validate
- * @returns True if valid, false otherwise
- */
-export function validateComplexity(complexity: number): boolean {
-  return complexity === 0 || complexity === 1 || complexity === 2 || complexity === 3;
-}

package/src/utils/json-utils.ts

@@ -356,81 +356,3 @@ export function extractJson<T = unknown>(
     error: 'No valid JSON found using any extraction method',
   };
 }
-
-/**
- * Result of string array normalization
- */
-export interface NormalizeArrayResult {
-  /** Normalized strings */
-  strings: string[];
-  /** Count of items that were skipped (empty after normalization) */
-  skippedCount: number;
-  /** Count of items that were objects and needed extraction */
-  extractedCount: number;
-  /** Warnings about any issues */
-  warnings: string[];
-}
-
-/**
- * Normalize string array with detailed feedback
- *
- * Normalizes an array of values to strings with detailed feedback
- * about what was normalized and any issues.
- *
- * @param arr - Array of values to normalize
- * @returns Detailed normalization result
- */
-export function normalizeStringArrayDetailed(
-  arr: unknown[]
-): NormalizeArrayResult {
-  const strings: string[] = [];
-  const warnings: string[] = [];
-  let skippedCount = 0;
-  let extractedCount = 0;
-
-  arr.forEach((item, index) => {
-    if (typeof item === 'string') {
-      const trimmed = item.trim();
-      if (trimmed.length > 0) {
-        strings.push(trimmed);
-      } else {
-        skippedCount++;
-        warnings.push(`Item ${index}: empty string`);
-      }
-    } else if (item !== null && typeof item === 'object') {
-      extractedCount++;
-      const obj = item as Record<string, unknown>;
-      const candidate = obj['query'] ?? obj['topic'] ?? obj['text'] ?? obj['task'];
-      
-      if (typeof candidate === 'string') {
-        const trimmed = candidate.trim();
-        if (trimmed.length > 0) {
-          strings.push(trimmed);
-        } else {
-          skippedCount++;
-          warnings.push(`Item ${index}: object extracted empty string`);
-        }
-      } else {
-        // No recognized key, stringify the whole object
-        const stringified = JSON.stringify(item);
-        if (stringified.length > 0) {
-          strings.push(stringified);
-          warnings.push(`Item ${index}: no recognized key, stringified object`);
-        } else {
-          skippedCount++;
-          warnings.push(`Item ${index}: object stringified to empty`);
-        }
-      }
-    } else {
-      const stringified = String(item).trim();
-      if (stringified.length > 0) {
-        strings.push(stringified);
-      } else {
-        skippedCount++;
-        warnings.push(`Item ${index}: ${typeof item} converted to empty string`);
-      }
-    }
-  });
-
-  return { strings, skippedCount, extractedCount, warnings };
-}

package/src/utils/log-utils.ts

@@ -91,9 +91,14 @@ const URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi
 // later one exposed. A cookie string is a ;-and-space separated list of secrets,
 // not a single token — mask the ENTIRE header value to end-of-line.
 const COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
-// key=value / key: "value" pairs whose key names a secret.
+// key=value / key: "value" pairs whose key names a secret. Includes bare
+// `key` (not just `api_key`/`private_key`): several provider query-param
+// APIs (e.g. StackExchange's `?key=<apikey>`) name the credential parameter
+// with no qualifying prefix. Over-redacting an unrelated "key" field (a
+// sort key, a map key) is the accepted trade-off — same one already made for
+// the equally generic bare `session`/`cookie` entries below.
 const SENSITIVE_KV_PATTERN =
-  /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
+  /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
 // Well-known opaque credential formats (OpenAI, GitHub, AWS, Slack, Anthropic).
 const KNOWN_TOKEN_PATTERN =
   /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
@@ -129,21 +134,13 @@ const CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
  * neither sink receives clear-text secrets or unbounded network data.
  */
 export function redactSecrets(message: string): string {
-  // slice() counts UTF-16 code units, so a naive cut at MAX_LOG_MESSAGE_LENGTH
-  // can land between the two halves of a surrogate pair (non-BMP characters —
-  // routine in scraped CJK/emoji content). The console sink then writes a
-  // lone surrogate, silently mangled to U+FFFD on UTF-8 encode; the JSON-file
-  // sink emits an unpaired \uXXXX escape that many strict JSON consumers
-  // reject outright. Same back-off truncateWithMarker (text-utils.ts) already
-  // applies for the same reason.
-  let keep = MAX_LOG_MESSAGE_LENGTH;
-  if (message.length > keep) {
-    const lastKeptUnit = message.charCodeAt(keep - 1);
-    if (lastKeptUnit >= 0xd800 && lastKeptUnit <= 0xdbff) keep -= 1;
-  }
-  let out = message.length > MAX_LOG_MESSAGE_LENGTH
-    ? `${message.slice(0, keep)}…[truncated ${message.length - keep} chars]`
-    : message;
+  // Redact BEFORE truncating. Every pattern below needs to see a secret in
+  // full to match it (e.g. LONG_HEX_SECRET_PATTERN requires 32+ contiguous
+  // hex chars) — truncating first can cut a secret mid-token, leaving
+  // whatever fragment survives the cut (e.g. the first 20 chars of a 64-hex
+  // browser auth secret) too short for any pattern to recognize, so it was
+  // written out in the clear right before the "…[truncated N chars]" marker.
+  let out = message;
   out = out.replace(ANSI_PATTERN, '');
   out = out.replace(URL_CREDENTIALS_PATTERN, '$1[REDACTED]@');
   // Redact whole-token formats BEFORE the key/value pass: the KV value class
@@ -161,6 +158,22 @@ export function redactSecrets(message: string): string {
   out = out.replace(KNOWN_TOKEN_PATTERN, '[REDACTED]');
   out = out.replace(PROVIDER_TOKEN_PATTERN, '[REDACTED]');
   out = out.replace(LONG_HEX_SECRET_PATTERN, '[REDACTED]');
+
+  // slice() counts UTF-16 code units, so a naive cut at MAX_LOG_MESSAGE_LENGTH
+  // can land between the two halves of a surrogate pair (non-BMP characters —
+  // routine in scraped CJK/emoji content). The console sink then writes a
+  // lone surrogate, silently mangled to U+FFFD on UTF-8 encode; the JSON-file
+  // sink emits an unpaired \uXXXX escape that many strict JSON consumers
+  // reject outright. Same back-off truncateWithMarker (text-utils.ts) already
+  // applies for the same reason.
+  let keep = MAX_LOG_MESSAGE_LENGTH;
+  if (out.length > keep) {
+    const lastKeptUnit = out.charCodeAt(keep - 1);
+    if (lastKeptUnit >= 0xd800 && lastKeptUnit <= 0xdbff) keep -= 1;
+  }
+  if (out.length > MAX_LOG_MESSAGE_LENGTH) {
+    out = `${out.slice(0, keep)}…[truncated ${out.length - keep} chars]`;
+  }
   return out;
 }
 
@@ -189,23 +202,9 @@ const TERMINAL_ESCAPE_PATTERN = new RegExp(
  * otherwise reach the user's terminal verbatim under PI_RESEARCH_CONSOLE_LOG.
  * The console sink keeps its own explicit CR/LF→"" replace (the form CodeQL
  * recognizes as the log-injection sanitizer); this handles everything else,
- * spacing out any leftover C0/DEL byte (BEL, backspace, …) like
- * neutralizeControlChars does.
+ * spacing out any leftover C0/DEL byte (BEL, backspace, …) via
+ * CONTROL_CHARS_PATTERN.
  */
 export function stripTerminalEscapes(message: string): string {
   return message.replace(TERMINAL_ESCAPE_PATTERN, '').replace(CONTROL_CHARS_PATTERN, ' ');
 }
-
-/**
- * Neutralize control characters (newlines, carriage returns, etc.) so that
- * untrusted content cannot forge or corrupt a raw console log line
- * (log-injection defense). Use for sinks that are NOT already structurally
- * escaped — the JSON log file escapes control chars on its own.
- */
-export function neutralizeControlChars(message: string): string {
-  // Strip CR/LF explicitly first. These are the actual log-injection vector
-  // (forged/corrupted log lines) and an explicit line-break replacement is the
-  // form static analysis recognizes as a log-injection sanitizer. The second
-  // pass removes any remaining C0 control chars + DEL.
-  return message.replace(/\r\n|\r|\n/g, ' ').replace(CONTROL_CHARS_PATTERN, ' ');
-}

package/src/utils/metrics.ts

@@ -198,14 +198,6 @@ export function runWithRunRegistry<T>(
   return runRegistryStorage.run(registry, fn);
 }
 
-/**
- * Returns the run-scoped registry if called inside a runWithRunRegistry context,
- * undefined otherwise.
- */
-export function getCurrentRunRegistry(): MetricsRegistry | undefined {
-  return runRegistryStorage.getStore();
-}
-
 // ── Session-level manager ──────────────────────────────────────────────────
 
 const MAX_RUN_HISTORY = 10;

package/src/utils/shared-links.ts

@@ -201,20 +201,6 @@ export function registerResearcherScrapes(researchId: string, researcherId: stri
     urls.forEach(url => set.add(normalizeUrl(url)));
 }
 
-/**
- * Check if a specific researcher scraped a given URL.
- */
-export function didResearcherScrape(researchId: string, researcherId: string, url: string): boolean {
-    return researcherScrapes.get(researchId)?.get(researcherId)?.has(normalizeUrl(url)) ?? false;
-}
-
-/**
- * Get all URLs scraped by a specific researcher.
- */
-export function getResearcherScrapes(researchId: string, researcherId: string): string[] {
-    return Array.from(researcherScrapes.get(researchId)?.get(researcherId) ?? []);
-}
-
 /**
  * Build the Session URL Pool footer for scrape responses.
  * Shows URLs previously scraped in the session, filtered to exclude the current batch.

package/src/utils/stdio-capture.ts

@@ -217,7 +217,21 @@ export async function captureStdio<T>(
     isAnyLoggerCapturingOutput = true;
   }
 
-  const decoder = new TextDecoder();
+  // Separate decoder PER logical byte stream, each used with {stream: true}.
+  // A single native write/writeSync call can split a multi-byte UTF-8
+  // character across two calls — {stream: true} is what lets a TextDecoder
+  // carry an incomplete trailing sequence over to the next decode() and
+  // reconstruct it, rather than replacing each half with U+FFFD independently
+  // (decode() with no options treats every call as a complete, final input).
+  // A SHARED decoder across genuinely independent streams would be its own
+  // bug even with {stream: true}: stdout, stderr, and raw FD1/FD2 writes can
+  // interleave, and a decoder's carried-over partial bytes are only valid
+  // when the next call continues the SAME byte stream — mixing streams would
+  // try to complete one stream's dangling sequence with another's bytes.
+  const stderrDecoder = new TextDecoder();
+  const stdoutDecoder = new TextDecoder();
+  const fd1SyncDecoder = new TextDecoder();
+  const fd2SyncDecoder = new TextDecoder();
 
   // Save originals
   const originalStderrWrite = process.stderr.write;
@@ -269,7 +283,18 @@ export async function captureStdio<T>(
   // Patch stderr.write
   (process.stderr.write as any) = (chunk: string | Uint8Array, encodingOrCb?: any, callback?: any) => {
     const cb = typeof encodingOrCb === 'function' ? encodingOrCb : callback;
-    const message = typeof chunk === 'string' ? chunk : decoder.decode(chunk);
+    const message = typeof chunk === 'string' ? chunk : stderrDecoder.decode(chunk, { stream: true });
+
+    // A chunk that lands entirely inside a multi-byte character's bytes
+    // decodes to '' with {stream: true} — the bytes are buffered inside
+    // stderrDecoder and will surface once the completing chunk arrives. This
+    // patch (unlike stdout's) always diverts rather than passing through, so
+    // without this guard it would write an empty log entry every time a
+    // native write happens to split mid-character.
+    if (message.length === 0) {
+      if (typeof cb === 'function') cb();
+      return true;
+    }
 
     // Check disk space before writing
     if (!hasSufficientDiskSpace()) {
@@ -294,7 +319,7 @@ export async function captureStdio<T>(
 
   // Patch stdout.write
   (process.stdout.write as any) = (chunk: string | Uint8Array, encodingOrCb?: any, callback?: any) => {
-    const message = typeof chunk === 'string' ? chunk : decoder.decode(chunk);
+    const message = typeof chunk === 'string' ? chunk : stdoutDecoder.decode(chunk, { stream: true });
     
     // Native log patterns or plain text that we want to divert from TUI
     const isNative = isNativeLog(message);
@@ -359,15 +384,27 @@ export async function captureStdio<T>(
     if (!descriptor || (descriptor.writable || descriptor.set)) {
       fs.writeSync = (fd: number, chunk: any, ...args: any[]) => {
         if (fd === 1 || fd === 2) {
-          const message = typeof chunk === 'string' ? chunk : decoder.decode(chunk);
-          
+          const fdDecoder = fd === 1 ? fd1SyncDecoder : fd2SyncDecoder;
+          const message = typeof chunk === 'string' ? chunk : fdDecoder.decode(chunk, { stream: true });
+
+          // FD 2 diverts unconditionally below (fd === 2 short-circuits
+          // shouldDivert), so unlike FD 1 — whose divert conditions already
+          // require non-empty content and naturally fall through to a
+          // passthrough write otherwise — an empty message here (a chunk that
+          // landed entirely inside a multi-byte character, buffered inside
+          // fdDecoder for the completing chunk) must be skipped explicitly to
+          // avoid writing an empty log entry on every mid-character split.
+          if (fd === 2 && message.length === 0) {
+            return (typeof chunk === 'string' ? Buffer.from(chunk).length : (chunk as any).length);
+          }
+
           const isNative = isNativeLog(message);
           const boxDrawing = isBoxDrawing(message);
           const complexTui = isComplexTui(message);
           const hasAnsi = message.includes('\x1b[');
 
-          const shouldDivert = fd === 2 || 
-            (isNative && !boxDrawing && !complexTui) || 
+          const shouldDivert = fd === 2 ||
+            (isNative && !boxDrawing && !complexTui) ||
             (!hasAnsi && !boxDrawing && message.trim().length > 0);
 
           if (shouldDivert) {

package/src/utils/url-utils.ts

@@ -6,6 +6,7 @@
  */
 
 import { logger } from '../logger.ts';
+import { redactSecrets } from './log-utils.ts';
 
 /**
  * Strips trailing punctuation an LLM commonly appends to a URL in prose
@@ -130,7 +131,12 @@ export function normalizeUrl(url: string): string {
     // Fallback for invalid URLs: aggressive best-effort normalization.
     // LLM hallucinations often produce broken URLs — log a warning so we can
     // trace them, then do our best to clean them for dedup purposes.
-    logger.debug(`[url-utils] normalizing unparseable URL (${url.slice(0, 200)}): ${_err instanceof Error ? _err.message : String(_err)}`);
+    // Redact BEFORE truncating: `url` can still carry userinfo credentials
+    // (https://user:pass@host/...) even when unparseable, and cutting at 200
+    // chars first can slice a credential mid-token before redactSecrets'
+    // patterns ever see the whole thing — same class of bug as the truncate-
+    // before-redact fix in log-utils.ts's redactSecrets itself.
+    logger.debug(`[url-utils] normalizing unparseable URL (${redactSecrets(url).slice(0, 200)}): ${_err instanceof Error ? _err.message : String(_err)}`);
     
     let cleaned = url.trim()
       // Strip leading/trailing markdown and punctuation often added by LLMs.

package/src/web-research/scraper-utils.ts

@@ -32,10 +32,14 @@ import { isCloudflareBlockError, isTaskTimeoutError, isPoolShutdownError } from
  * logger.error's error-tracker re-track, the diagnostic error count) focused on
  * genuine faults. The messages matched here are produced by validateContent
  * (`Fetch blocked:` / `Fetch returned stub:`), scrapeWithFetch (`HTTP <code>`)
- * and extractPdfToMarkdown (`Could not extract content from PDF`) — a
- * malformed/encrypted/scanned-image-only PDF is routine on the open web and
+ * and extractPdfToMarkdown (`Could not extract content from PDF`, `PDF too
+ * large`) — a malformed/encrypted/scanned-image-only PDF, or one that simply
+ * exceeds the deliberate 100MB policy cap, is routine on the open web and
  * just as non-actionable as a stub page or a 404; the remaining cases
- * delegate to the shared browser-layer predicates.
+ * delegate to the shared browser-layer predicates. Deliberately does NOT
+ * match 'PDF extraction unavailable' (a pdf-oxide-wasm native-module load
+ * failure) — that is an infrastructure fault affecting every URL, not a
+ * routine per-PDF outcome, and must still surface at ERROR.
  */
 export function isBenignScrapeFailure(error: unknown): boolean {
   const msg = error instanceof Error ? error.message : String(error);
@@ -43,6 +47,7 @@ export function isBenignScrapeFailure(error: unknown): boolean {
     msg.startsWith('Fetch blocked:') ||                  // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
     msg.startsWith('Fetch returned stub:') ||             // nav-only / near-empty page
     msg.startsWith('Could not extract content from PDF') || // malformed/encrypted/scanned-only PDF
+    msg.startsWith('PDF too large') ||                    // exceeds the deliberate 100MB policy cap
     /^HTTP \d{3}\b/.test(msg) ||                          // remote 4xx/5xx status
     isCloudflareBlockError(error) ||
     isTaskTimeoutError(error) ||

package/src/web-research/web-scraper.ts

@@ -88,8 +88,40 @@ async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
   }
 
   const pdfExtractionStart = Date.now();
+
+  // The native module load is caught SEPARATELY from the parse/extraction
+  // logic below: a failure here (missing prebuilt binary for this platform,
+  // corrupted install) is an INFRASTRUCTURE fault — PDF extraction is broken
+  // for every URL, not just this one — not the routine, expected outcome of
+  // a malformed/encrypted/scanned-image-only PDF. Sharing one catch with the
+  // parse logic used to log both at `debug` and lump both under
+  // 'extraction_failed', so a broken install produced an endless stream of
+  // silent per-PDF "failures" instead of ever surfacing the real cause.
+  let WasmPdfDocument: (new (bytes: Uint8Array) => {
+    pageCount(): number;
+    toMarkdownAll(): string;
+    toMarkdown(page: number): string;
+    free(): void;
+  });
+  try {
+    ({ WasmPdfDocument } = await import('pdf-oxide-wasm'));
+  } catch (e) {
+    const msg = e instanceof Error ? e.message : String(e);
+    logger.error(`[Scrapers] pdf-oxide-wasm failed to load — PDF extraction is unavailable (not just for this URL): ${msg}`);
+    metrics.increment('scrape_pdf_errors_total', 1, { error_type: 'native_module_unavailable' });
+    errorTracker.trackError(e instanceof Error ? e : String(e), {
+      component: 'scrapers',
+      operation: 'pdf-extract',
+      contentType: 'pdf',
+      errorType: 'native_module_unavailable',
+    });
+    // Deliberately does NOT start with 'Could not extract content from PDF' —
+    // isBenignScrapeFailure must NOT classify this as the routine per-PDF
+    // outcome it is for the parse-failure branch below.
+    throw new Error(`PDF extraction unavailable (pdf-oxide-wasm failed to load: ${msg})`, { cause: e });
+  }
+
   try {
-    const { WasmPdfDocument } = await import('pdf-oxide-wasm');
     const doc = new WasmPdfDocument(bytes);
     const pageCount = doc.pageCount();