@lincoln504/pi-research 1.3.3 → 1.3.5

package/src/utils/log-utils.ts

@@ -91,9 +91,14 @@ const URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi
 // later one exposed. A cookie string is a ;-and-space separated list of secrets,
 // not a single token — mask the ENTIRE header value to end-of-line.
 const COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
-// key=value / key: "value" pairs whose key names a secret.
+// key=value / key: "value" pairs whose key names a secret. Includes bare
+// `key` (not just `api_key`/`private_key`): several provider query-param
+// APIs (e.g. StackExchange's `?key=<apikey>`) name the credential parameter
+// with no qualifying prefix. Over-redacting an unrelated "key" field (a
+// sort key, a map key) is the accepted trade-off — same one already made for
+// the equally generic bare `session`/`cookie` entries below.
 const SENSITIVE_KV_PATTERN =
-  /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
+  /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
 // Well-known opaque credential formats (OpenAI, GitHub, AWS, Slack, Anthropic).
 const KNOWN_TOKEN_PATTERN =
   /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
@@ -128,22 +133,25 @@ const CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
  * Applied to every message before it reaches the log file or the console, so
  * neither sink receives clear-text secrets or unbounded network data.
  */
+// How much raw input the regex chain below is allowed to scan before a final
+// truncation to MAX_LOG_MESSAGE_LENGTH. A multiple of MAX_LOG_MESSAGE_LENGTH,
+// not 1:1, because most patterns below SHRINK text (a 64-char secret becomes
+// "[REDACTED]"), so more than `keep` raw characters can be needed to fully
+// populate the final truncated output. Nothing past this point can ever
+// survive into the visible output regardless, so skipping it is free — this
+// is what bounds redactSecrets() to O(MAX_LOG_MESSAGE_LENGTH) instead of
+// O(input length) on multi-megabyte scraped payloads.
+const REDACT_SCAN_LENGTH = MAX_LOG_MESSAGE_LENGTH * 4;
+
 export function redactSecrets(message: string): string {
-  // slice() counts UTF-16 code units, so a naive cut at MAX_LOG_MESSAGE_LENGTH
-  // can land between the two halves of a surrogate pair (non-BMP characters —
-  // routine in scraped CJK/emoji content). The console sink then writes a
-  // lone surrogate, silently mangled to U+FFFD on UTF-8 encode; the JSON-file
-  // sink emits an unpaired \uXXXX escape that many strict JSON consumers
-  // reject outright. Same back-off truncateWithMarker (text-utils.ts) already
-  // applies for the same reason.
-  let keep = MAX_LOG_MESSAGE_LENGTH;
-  if (message.length > keep) {
-    const lastKeptUnit = message.charCodeAt(keep - 1);
-    if (lastKeptUnit >= 0xd800 && lastKeptUnit <= 0xdbff) keep -= 1;
-  }
-  let out = message.length > MAX_LOG_MESSAGE_LENGTH
-    ? `${message.slice(0, keep)}…[truncated ${message.length - keep} chars]`
-    : message;
+  // Redact BEFORE truncating. Every pattern below needs to see a secret in
+  // full to match it (e.g. LONG_HEX_SECRET_PATTERN requires 32+ contiguous
+  // hex chars) — truncating first can cut a secret mid-token, leaving
+  // whatever fragment survives the cut (e.g. the first 20 chars of a 64-hex
+  // browser auth secret) too short for any pattern to recognize, so it was
+  // written out in the clear right before the "…[truncated N chars]" marker.
+  const rawLength = message.length;
+  let out = rawLength > REDACT_SCAN_LENGTH ? message.slice(0, REDACT_SCAN_LENGTH) : message;
   out = out.replace(ANSI_PATTERN, '');
   out = out.replace(URL_CREDENTIALS_PATTERN, '$1[REDACTED]@');
   // Redact whole-token formats BEFORE the key/value pass: the KV value class
@@ -161,6 +169,22 @@ export function redactSecrets(message: string): string {
   out = out.replace(KNOWN_TOKEN_PATTERN, '[REDACTED]');
   out = out.replace(PROVIDER_TOKEN_PATTERN, '[REDACTED]');
   out = out.replace(LONG_HEX_SECRET_PATTERN, '[REDACTED]');
+
+  // slice() counts UTF-16 code units, so a naive cut at MAX_LOG_MESSAGE_LENGTH
+  // can land between the two halves of a surrogate pair (non-BMP characters —
+  // routine in scraped CJK/emoji content). The console sink then writes a
+  // lone surrogate, silently mangled to U+FFFD on UTF-8 encode; the JSON-file
+  // sink emits an unpaired \uXXXX escape that many strict JSON consumers
+  // reject outright. Same back-off truncateWithMarker (text-utils.ts) already
+  // applies for the same reason.
+  let keep = MAX_LOG_MESSAGE_LENGTH;
+  if (out.length > keep) {
+    const lastKeptUnit = out.charCodeAt(keep - 1);
+    if (lastKeptUnit >= 0xd800 && lastKeptUnit <= 0xdbff) keep -= 1;
+  }
+  if (out.length > MAX_LOG_MESSAGE_LENGTH) {
+    out = `${out.slice(0, keep)}…[truncated ${rawLength - keep} chars]`;
+  }
   return out;
 }
 
@@ -189,23 +213,9 @@ const TERMINAL_ESCAPE_PATTERN = new RegExp(
  * otherwise reach the user's terminal verbatim under PI_RESEARCH_CONSOLE_LOG.
  * The console sink keeps its own explicit CR/LF→"" replace (the form CodeQL
  * recognizes as the log-injection sanitizer); this handles everything else,
- * spacing out any leftover C0/DEL byte (BEL, backspace, …) like
- * neutralizeControlChars does.
+ * spacing out any leftover C0/DEL byte (BEL, backspace, …) via
+ * CONTROL_CHARS_PATTERN.
  */
 export function stripTerminalEscapes(message: string): string {
   return message.replace(TERMINAL_ESCAPE_PATTERN, '').replace(CONTROL_CHARS_PATTERN, ' ');
 }
-
-/**
- * Neutralize control characters (newlines, carriage returns, etc.) so that
- * untrusted content cannot forge or corrupt a raw console log line
- * (log-injection defense). Use for sinks that are NOT already structurally
- * escaped — the JSON log file escapes control chars on its own.
- */
-export function neutralizeControlChars(message: string): string {
-  // Strip CR/LF explicitly first. These are the actual log-injection vector
-  // (forged/corrupted log lines) and an explicit line-break replacement is the
-  // form static analysis recognizes as a log-injection sanitizer. The second
-  // pass removes any remaining C0 control chars + DEL.
-  return message.replace(/\r\n|\r|\n/g, ' ').replace(CONTROL_CHARS_PATTERN, ' ');
-}

package/src/utils/metrics.ts

@@ -198,14 +198,6 @@ export function runWithRunRegistry<T>(
   return runRegistryStorage.run(registry, fn);
 }
 
-/**
- * Returns the run-scoped registry if called inside a runWithRunRegistry context,
- * undefined otherwise.
- */
-export function getCurrentRunRegistry(): MetricsRegistry | undefined {
-  return runRegistryStorage.getStore();
-}
-
 // ── Session-level manager ──────────────────────────────────────────────────
 
 const MAX_RUN_HISTORY = 10;

package/src/utils/shared-links.ts

@@ -201,20 +201,6 @@ export function registerResearcherScrapes(researchId: string, researcherId: stri
     urls.forEach(url => set.add(normalizeUrl(url)));
 }
 
-/**
- * Check if a specific researcher scraped a given URL.
- */
-export function didResearcherScrape(researchId: string, researcherId: string, url: string): boolean {
-    return researcherScrapes.get(researchId)?.get(researcherId)?.has(normalizeUrl(url)) ?? false;
-}
-
-/**
- * Get all URLs scraped by a specific researcher.
- */
-export function getResearcherScrapes(researchId: string, researcherId: string): string[] {
-    return Array.from(researcherScrapes.get(researchId)?.get(researcherId) ?? []);
-}
-
 /**
  * Build the Session URL Pool footer for scrape responses.
  * Shows URLs previously scraped in the session, filtered to exclude the current batch.

package/src/utils/stdio-capture.ts

@@ -217,7 +217,21 @@ export async function captureStdio<T>(
     isAnyLoggerCapturingOutput = true;
   }
 
-  const decoder = new TextDecoder();
+  // Separate decoder PER logical byte stream, each used with {stream: true}.
+  // A single native write/writeSync call can split a multi-byte UTF-8
+  // character across two calls — {stream: true} is what lets a TextDecoder
+  // carry an incomplete trailing sequence over to the next decode() and
+  // reconstruct it, rather than replacing each half with U+FFFD independently
+  // (decode() with no options treats every call as a complete, final input).
+  // A SHARED decoder across genuinely independent streams would be its own
+  // bug even with {stream: true}: stdout, stderr, and raw FD1/FD2 writes can
+  // interleave, and a decoder's carried-over partial bytes are only valid
+  // when the next call continues the SAME byte stream — mixing streams would
+  // try to complete one stream's dangling sequence with another's bytes.
+  const stderrDecoder = new TextDecoder();
+  const stdoutDecoder = new TextDecoder();
+  const fd1SyncDecoder = new TextDecoder();
+  const fd2SyncDecoder = new TextDecoder();
 
   // Save originals
   const originalStderrWrite = process.stderr.write;
@@ -269,7 +283,18 @@ export async function captureStdio<T>(
   // Patch stderr.write
   (process.stderr.write as any) = (chunk: string | Uint8Array, encodingOrCb?: any, callback?: any) => {
     const cb = typeof encodingOrCb === 'function' ? encodingOrCb : callback;
-    const message = typeof chunk === 'string' ? chunk : decoder.decode(chunk);
+    const message = typeof chunk === 'string' ? chunk : stderrDecoder.decode(chunk, { stream: true });
+
+    // A chunk that lands entirely inside a multi-byte character's bytes
+    // decodes to '' with {stream: true} — the bytes are buffered inside
+    // stderrDecoder and will surface once the completing chunk arrives. This
+    // patch (unlike stdout's) always diverts rather than passing through, so
+    // without this guard it would write an empty log entry every time a
+    // native write happens to split mid-character.
+    if (message.length === 0) {
+      if (typeof cb === 'function') cb();
+      return true;
+    }
 
     // Check disk space before writing
     if (!hasSufficientDiskSpace()) {
@@ -294,7 +319,7 @@ export async function captureStdio<T>(
 
   // Patch stdout.write
   (process.stdout.write as any) = (chunk: string | Uint8Array, encodingOrCb?: any, callback?: any) => {
-    const message = typeof chunk === 'string' ? chunk : decoder.decode(chunk);
+    const message = typeof chunk === 'string' ? chunk : stdoutDecoder.decode(chunk, { stream: true });
     
     // Native log patterns or plain text that we want to divert from TUI
     const isNative = isNativeLog(message);
@@ -359,15 +384,27 @@ export async function captureStdio<T>(
     if (!descriptor || (descriptor.writable || descriptor.set)) {
       fs.writeSync = (fd: number, chunk: any, ...args: any[]) => {
         if (fd === 1 || fd === 2) {
-          const message = typeof chunk === 'string' ? chunk : decoder.decode(chunk);
-          
+          const fdDecoder = fd === 1 ? fd1SyncDecoder : fd2SyncDecoder;
+          const message = typeof chunk === 'string' ? chunk : fdDecoder.decode(chunk, { stream: true });
+
+          // FD 2 diverts unconditionally below (fd === 2 short-circuits
+          // shouldDivert), so unlike FD 1 — whose divert conditions already
+          // require non-empty content and naturally fall through to a
+          // passthrough write otherwise — an empty message here (a chunk that
+          // landed entirely inside a multi-byte character, buffered inside
+          // fdDecoder for the completing chunk) must be skipped explicitly to
+          // avoid writing an empty log entry on every mid-character split.
+          if (fd === 2 && message.length === 0) {
+            return (typeof chunk === 'string' ? Buffer.from(chunk).length : (chunk as any).length);
+          }
+
           const isNative = isNativeLog(message);
           const boxDrawing = isBoxDrawing(message);
           const complexTui = isComplexTui(message);
           const hasAnsi = message.includes('\x1b[');
 
-          const shouldDivert = fd === 2 || 
-            (isNative && !boxDrawing && !complexTui) || 
+          const shouldDivert = fd === 2 ||
+            (isNative && !boxDrawing && !complexTui) ||
             (!hasAnsi && !boxDrawing && message.trim().length > 0);
 
           if (shouldDivert) {

package/src/utils/text-utils.ts

@@ -188,8 +188,12 @@ export function formatDuration(ms: number): string {
   if (ms < 1000) return `${ms}ms`;
   const seconds = ms / 1000;
   if (seconds < 60) return `${seconds.toFixed(1)}s`;
-  const minutes = Math.floor(seconds / 60);
-  const remainingSeconds = Math.round(seconds % 60);
+  // Round the total first, then derive minutes/seconds from that integer —
+  // rounding remainingSeconds independently of minutes let e.g. 59.6s round up
+  // to a nonsensical "Xm 60s" instead of rolling over to the next minute.
+  const totalSeconds = Math.round(seconds);
+  const minutes = Math.floor(totalSeconds / 60);
+  const remainingSeconds = totalSeconds % 60;
   if (minutes < 60) return `${minutes}m ${remainingSeconds}s`;
   const hours = Math.floor(minutes / 60);
   const remainingMinutes = minutes % 60;

package/src/utils/url-utils.ts

@@ -6,6 +6,7 @@
  */
 
 import { logger } from '../logger.ts';
+import { redactSecrets } from './log-utils.ts';
 
 /**
  * Strips trailing punctuation an LLM commonly appends to a URL in prose
@@ -130,7 +131,12 @@ export function normalizeUrl(url: string): string {
     // Fallback for invalid URLs: aggressive best-effort normalization.
     // LLM hallucinations often produce broken URLs — log a warning so we can
     // trace them, then do our best to clean them for dedup purposes.
-    logger.debug(`[url-utils] normalizing unparseable URL (${url.slice(0, 200)}): ${_err instanceof Error ? _err.message : String(_err)}`);
+    // Redact BEFORE truncating: `url` can still carry userinfo credentials
+    // (https://user:pass@host/...) even when unparseable, and cutting at 200
+    // chars first can slice a credential mid-token before redactSecrets'
+    // patterns ever see the whole thing — same class of bug as the truncate-
+    // before-redact fix in log-utils.ts's redactSecrets itself.
+    logger.debug(`[url-utils] normalizing unparseable URL (${redactSecrets(url).slice(0, 200)}): ${_err instanceof Error ? _err.message : String(_err)}`);
     
     let cleaned = url.trim()
       // Strip leading/trailing markdown and punctuation often added by LLMs.

package/src/web-research/scraper-utils.ts

@@ -32,10 +32,14 @@ import { isCloudflareBlockError, isTaskTimeoutError, isPoolShutdownError } from
  * logger.error's error-tracker re-track, the diagnostic error count) focused on
  * genuine faults. The messages matched here are produced by validateContent
  * (`Fetch blocked:` / `Fetch returned stub:`), scrapeWithFetch (`HTTP <code>`)
- * and extractPdfToMarkdown (`Could not extract content from PDF`) — a
- * malformed/encrypted/scanned-image-only PDF is routine on the open web and
+ * and extractPdfToMarkdown (`Could not extract content from PDF`, `PDF too
+ * large`) — a malformed/encrypted/scanned-image-only PDF, or one that simply
+ * exceeds the deliberate 100MB policy cap, is routine on the open web and
  * just as non-actionable as a stub page or a 404; the remaining cases
- * delegate to the shared browser-layer predicates.
+ * delegate to the shared browser-layer predicates. Deliberately does NOT
+ * match 'PDF extraction unavailable' (a pdf-oxide-wasm native-module load
+ * failure) — that is an infrastructure fault affecting every URL, not a
+ * routine per-PDF outcome, and must still surface at ERROR.
  */
 export function isBenignScrapeFailure(error: unknown): boolean {
   const msg = error instanceof Error ? error.message : String(error);
@@ -43,6 +47,7 @@ export function isBenignScrapeFailure(error: unknown): boolean {
     msg.startsWith('Fetch blocked:') ||                  // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
     msg.startsWith('Fetch returned stub:') ||             // nav-only / near-empty page
     msg.startsWith('Could not extract content from PDF') || // malformed/encrypted/scanned-only PDF
+    msg.startsWith('PDF too large') ||                    // exceeds the deliberate 100MB policy cap
     /^HTTP \d{3}\b/.test(msg) ||                          // remote 4xx/5xx status
     isCloudflareBlockError(error) ||
     isTaskTimeoutError(error) ||

package/src/web-research/web-scraper.ts

@@ -88,8 +88,40 @@ async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
   }
 
   const pdfExtractionStart = Date.now();
+
+  // The native module load is caught SEPARATELY from the parse/extraction
+  // logic below: a failure here (missing prebuilt binary for this platform,
+  // corrupted install) is an INFRASTRUCTURE fault — PDF extraction is broken
+  // for every URL, not just this one — not the routine, expected outcome of
+  // a malformed/encrypted/scanned-image-only PDF. Sharing one catch with the
+  // parse logic used to log both at `debug` and lump both under
+  // 'extraction_failed', so a broken install produced an endless stream of
+  // silent per-PDF "failures" instead of ever surfacing the real cause.
+  let WasmPdfDocument: (new (bytes: Uint8Array) => {
+    pageCount(): number;
+    toMarkdownAll(): string;
+    toMarkdown(page: number): string;
+    free(): void;
+  });
+  try {
+    ({ WasmPdfDocument } = await import('pdf-oxide-wasm'));
+  } catch (e) {
+    const msg = e instanceof Error ? e.message : String(e);
+    logger.error(`[Scrapers] pdf-oxide-wasm failed to load — PDF extraction is unavailable (not just for this URL): ${msg}`);
+    metrics.increment('scrape_pdf_errors_total', 1, { error_type: 'native_module_unavailable' });
+    errorTracker.trackError(e instanceof Error ? e : String(e), {
+      component: 'scrapers',
+      operation: 'pdf-extract',
+      contentType: 'pdf',
+      errorType: 'native_module_unavailable',
+    });
+    // Deliberately does NOT start with 'Could not extract content from PDF' —
+    // isBenignScrapeFailure must NOT classify this as the routine per-PDF
+    // outcome it is for the parse-failure branch below.
+    throw new Error(`PDF extraction unavailable (pdf-oxide-wasm failed to load: ${msg})`, { cause: e });
+  }
+
   try {
-    const { WasmPdfDocument } = await import('pdf-oxide-wasm');
     const doc = new WasmPdfDocument(bytes);
     const pageCount = doc.pageCount();
     

package/src/youtube/potoken.ts

@@ -42,6 +42,7 @@
  */
 
 import { logger } from '../logger.ts';
+import { safeUnref } from '../utils/safe-unref.ts';
 import { metrics } from '../utils/metrics.ts';
 
 /**
@@ -76,6 +77,15 @@ function describeError(error: unknown): string {
 // Module-level mutex: serialize all globalThis mutation across concurrent callers.
 let mintChain: Promise<unknown> = Promise.resolve();
 
+// How long the mutex waits for a straggling doMint() before force-releasing
+// the chain for the next caller. doMint() awaits non-fetch operations with no
+// abort signal (interpreter eval, BotGuard VM execution against a
+// hostile/malformed attestation script) that could in principle hang forever;
+// without this bound, a single wedged call would permanently break YouTube
+// transcript fetching for the rest of the process. Mirrors the hard-release
+// timer on the browser page-creation mutex in thread-worker-messaging.ts.
+const MINT_HARD_RELEASE_MS = 5 * 60 * 1000;
+
 /**
  * Mint a session-bound token (for `identifiers[0]`, the visitorData) plus one
  * content-bound token per remaining identifier (the video IDs), all from a
@@ -90,8 +100,22 @@ export async function mintPoTokens(
 ): Promise<Map<string, string>> {
   // Chain on the mutex so only one bridged section runs at a time.
   const run = mintChain.then(() => doMint(identifiers, opts), () => doMint(identifiers, opts));
-  // Keep the chain alive regardless of this call's success/failure.
-  mintChain = run.then(() => undefined, () => undefined);
+  // Keep the chain alive regardless of this call's success/failure, but bound
+  // how long the NEXT caller waits on it: if `run` never settles, force the
+  // chain open after the hard timeout rather than wedging every future
+  // mintPoTokens() call behind it forever. The abandoned `run` may still
+  // mutate globalThis in the background after this — an accepted trade-off
+  // in the pathological case, same as the sibling browser-worker fix.
+  const settled = run.then(() => undefined, () => undefined);
+  let hardTimer: NodeJS.Timeout;
+  const hardRelease = new Promise<void>(resolve => {
+    hardTimer = setTimeout(() => {
+      logger.warn('[youtube] potoken mint mutex force-released — doMint() did not settle within the hard timeout');
+      resolve();
+    }, MINT_HARD_RELEASE_MS);
+    safeUnref(hardTimer);
+  });
+  mintChain = Promise.race([settled, hardRelease]).finally(() => clearTimeout(hardTimer));
   return run;
 }
 

package/src/youtube/transcript-client.ts

@@ -20,6 +20,7 @@
 import { withTimeout, retryWithBackoff, createTimeoutSignal } from '../web-research/retry-utils.ts';
 import { CircuitBreaker } from '../utils/circuit-breaker.ts';
 import { logger } from '../logger.ts';
+import { redactSecrets } from '../utils/log-utils.ts';
 import { metrics } from '../utils/metrics.ts';
 import { readTextCapped } from '../utils/http-body.ts';
 import { mintPoTokens } from './potoken.ts';
@@ -355,8 +356,10 @@ function parseJson3(body: string): string {
     // Log the real cause: an unparseable body is NOT necessarily an empty
     // bot-protection response (the caller's default diagnosis) — it can be an HTML
     // interstitial or a response-shape change. A short snippet disambiguates without
-    // dumping a huge/again-sensitive body.
-    logger.debug(`[youtube] timedtext body was not valid json3 (first 120 chars): ${body.slice(0, 120)}`);
+    // dumping a huge/again-sensitive body. Redact BEFORE truncating to 120 chars —
+    // slicing first can cut a secret-shaped token mid-way, leaving a fragment too
+    // short for redactSecrets' whole-token patterns to recognize.
+    logger.debug(`[youtube] timedtext body was not valid json3 (first 120 chars): ${redactSecrets(body).slice(0, 120)}`);
     return '';
   }
   const events = (parsed as { events?: unknown[] }).events;