@lincoln504/pi-research 1.3.4 → 1.3.6

package/src/tools/scrape.ts

@@ -24,6 +24,7 @@ import {
   MAX_SCRAPE_CONTENT_CHARS_PER_DOC,
 } from '../constants.ts';
 import { truncateWithMarker } from '../utils/text-utils.ts';
+import { redactSecrets } from '../utils/log-utils.ts';
 import { type Config, getConfig } from '../config.ts';
 import { getService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
 import { ServiceNames } from '../core/service-interfaces.ts';
@@ -261,16 +262,17 @@ export function createScrapeTool(options: {
       const concurrency = p.maxConcurrency || defaultConcurrency;
       
       try {
-        const results = await scrape(urlsToFetch, concurrency, signal, options.config, getGlobalState().researchId, (result) => {
+        const results = await scrape(urlsToFetch, concurrency, signal, config, getGlobalState().researchId, (result) => {
           options.onUrlScrapeResult?.(result.url, result.success);
         }, container);
         freshResults = Array.isArray(results) ? results : [];
       } catch (error) {
-          logger.error(`[scrape tool] Scrape failed: ${error instanceof Error ? error.message : String(error)}`);
+          const message = error instanceof Error ? error.message : String(error);
+          logger.error(`[scrape tool] Scrape failed: ${message}`);
           metrics.increment('tool_scrape_calls_total', 1, { status: 'error' });
           return {
-            content: [{ type: 'text', text: `# Scrape Failed\n\n${error instanceof Error ? error.message : String(error)}` }],
-            details: { error: String(error) },
+            content: [{ type: 'text', text: `# Scrape Failed\n\n${redactSecrets(message)}` }],
+            details: { error: redactSecrets(String(error)) },
           };
       }
 

package/src/tools/security.ts

@@ -13,6 +13,15 @@ import { getMaxGatheringCalls } from '../constants.ts';
 import { metrics } from '../utils/metrics.ts';
 import type { SecuritySearchParams } from '../security/types.ts';
 
+const MAX_DESCRIPTION_CHARS = 300;
+
+/** Truncates an advisory description for the markdown summary line. */
+function truncateDescription(description: string): string {
+  return description.length > MAX_DESCRIPTION_CHARS
+    ? `${description.substring(0, MAX_DESCRIPTION_CHARS)}...`
+    : description;
+}
+
 export function createSecuritySearchTool(options: {
   ctx: ExtensionContext;
   tracker: ToolUsageTracker;
@@ -229,7 +238,7 @@ export function createSecuritySearchTool(options: {
               }
             }
             const description = vuln.description;
-            markdown += `- **Description:** ${description.length > 300 ? `${description.substring(0, 300)}...` : description}\n`;
+            markdown += `- **Description:** ${truncateDescription(description)}\n`;
             if (vuln.knownExploited === true) {
               markdown += '- **[Actively Exploited]**\n';
             }
@@ -270,7 +279,7 @@ export function createSecuritySearchTool(options: {
               markdown += `- **Product:** ${vuln.product}\n`;
             }
             const description = vuln.description;
-            markdown += `- **Description:** ${description.length > 300 ? `${description.substring(0, 300)}...` : description}\n`;
+            markdown += `- **Description:** ${truncateDescription(description)}\n`;
             if (vuln.dueDate !== undefined) {
               markdown += `- **Due Date:** ${vuln.dueDate}\n`;
             }
@@ -310,7 +319,7 @@ export function createSecuritySearchTool(options: {
             }
             markdown += `- **Summary:** ${adv.summary}\n`;
             const description = adv.description ?? '';
-            markdown += `- **Description:** ${description.length > 300 ? `${description.substring(0, 300)}...` : description}\n`;
+            markdown += `- **Description:** ${truncateDescription(description)}\n`;
             if (adv.published) {
               markdown += `- **Published:** ${adv.published}\n`;
             }
@@ -344,7 +353,7 @@ export function createSecuritySearchTool(options: {
             markdown += `### ${vuln.id}\n`;
             markdown += `- **Severity:** ${vuln.severity}\n`;
             const description = vuln.description;
-            markdown += `- **Description:** ${description.length > 300 ? `${description.substring(0, 300)}...` : description}\n`;
+            markdown += `- **Description:** ${truncateDescription(description)}\n`;
             if (vuln.affectedProducts && vuln.affectedProducts.length > 0) {
               markdown += `- **Affected:** ${vuln.affectedProducts.join(', ')}\n`;
             }

package/src/utils/log-rotation.ts

@@ -10,6 +10,40 @@ export interface RotationLogger {
   log(...args: unknown[]): void;
 }
 
+// Numeric equivalent of the 'a' flag string (O_WRONLY|O_APPEND|O_CREAT), with
+// O_NOFOLLOW added when the platform defines it (not available on Windows) so
+// the open refuses to follow a symlink an attacker with write access to the
+// log directory could plant at this path in the window between the preceding
+// unlink/rename and this reopen — defense-in-depth on top of an already-narrow
+// threat model (it requires local write access to this directory to begin with).
+const APPEND_CREATE_NOFOLLOW_FLAGS =
+  fs.constants.O_WRONLY | fs.constants.O_APPEND | fs.constants.O_CREAT |
+  (typeof fs.constants.O_NOFOLLOW === 'number' ? fs.constants.O_NOFOLLOW : 0);
+
+/** Best-effort (re)create `filePath` with `mode`, refusing to follow a symlink
+ * planted at the path, and unconditionally stamp `mode` on it. Uses fchmodSync
+ * on the already-open (and O_NOFOLLOW-verified) descriptor rather than
+ * fs.chmodSync(filePath, ...) — chmodSync operates on the path and follows
+ * symlinks, which would silently re-open the exact hole O_NOFOLLOW closes if
+ * a symlink appears at this path between the open above and a path-based
+ * chmod. mode is only applied by open() when O_CREAT actually creates the
+ * file, so callers relying on the file always ending up at `mode` (even when
+ * a concurrent writer already recreated it) still need this explicit stamp.
+ * Failures are swallowed by design — see call sites.
+ * Exported for unit testing of the symlink-refusal behavior. */
+export function touchOwnerOnly(filePath: string, mode: number): void {
+  let fd: number | undefined;
+  try {
+    fd = fs.openSync(filePath, APPEND_CREATE_NOFOLLOW_FLAGS, mode);
+    fs.fchmodSync(fd, mode);
+  } catch { /* best-effort — the next append recreates the file */ }
+  finally {
+    if (fd !== undefined) {
+      try { fs.closeSync(fd); } catch { /* best-effort */ }
+    }
+  }
+}
+
 export class LogRotation {
   private readonly MAX_LOG_SIZE = 10 * 1024 * 1024; // 10MB max file size
   private readonly MAX_LOG_FILES = 10; // Keep last 10 archived logs
@@ -29,15 +63,11 @@ export class LogRotation {
         fs.unlinkSync(logFile);
         // Same rationale as the post-rotation recreate in rotateLogFile: the
         // unlink discarded the 0o600 file, and a umask-default recreate by the
-        // next append would drop the owner-only posture.
-        try {
-          fs.closeSync(fs.openSync(logFile, 'a', 0o600));
-        } catch { /* best-effort — the next append recreates the file */ }
-        // And, as there, re-stamp the mode: a concurrent append landing between
-        // the unlink and the open makes the creation-time 0o600 a no-op.
-        try {
-          fs.chmodSync(logFile, 0o600);
-        } catch { /* best-effort */ }
+        // next append would drop the owner-only posture. touchOwnerOnly stamps
+        // the mode unconditionally (via fchmod on its own fd), covering the
+        // case where a concurrent append already recreated the file at umask
+        // perms between the unlink and this call.
+        touchOwnerOnly(logFile, 0o600);
       }
 
       const files = fs.readdirSync(logDir);
@@ -92,20 +122,12 @@ export class LogRotation {
       // Rotation moved the 0o600 file aside; without an explicit recreate the next
       // append materializes it at the umask default (typically 0644) in a
       // world-traversable tmpdir, silently dropping the owner-only posture Logger
-      // establishes at construction. Mode applies only at creation, so this is
-      // harmless where a file already exists or mode semantics differ (win32).
-      try {
-        fs.closeSync(fs.openSync(logFile, 'a', 0o600));
-      } catch { /* best-effort — the next append recreates the file */ }
-      // The mode above applies only if THIS open created the file. Several
-      // processes append to this consolidated log, so a concurrent async append
-      // can recreate it at umask perms between the rename and the open, turning
-      // the 0o600 into a no-op — stamp the mode unconditionally to close that
-      // window for good. ENOENT (nothing recreated it and the open failed) is
+      // establishes at construction. touchOwnerOnly stamps the mode
+      // unconditionally (via fchmod on its own fd) regardless of whether this
+      // open created the file or a concurrent append already recreated it
+      // between the rename and this call — ENOENT (open itself failed) is
       // tolerable: best-effort posture, same as the recreate itself.
-      try {
-        fs.chmodSync(logFile, 0o600);
-      } catch { /* best-effort */ }
+      touchOwnerOnly(logFile, 0o600);
 
 
       // Clean up old archives

package/src/utils/log-utils.ts

@@ -133,6 +133,25 @@ const CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
  * Applied to every message before it reaches the log file or the console, so
  * neither sink receives clear-text secrets or unbounded network data.
  */
+// How much raw input the regex chain below is allowed to scan before a final
+// truncation to MAX_LOG_MESSAGE_LENGTH. A multiple of MAX_LOG_MESSAGE_LENGTH,
+// not 1:1, because most patterns below SHRINK text (a 64-char secret becomes
+// "[REDACTED]"), so more than `keep` raw characters can be needed to fully
+// populate the final truncated output. Nothing past this point can ever
+// survive into the visible output regardless, so skipping it is free — this
+// is what bounds redactSecrets() to O(MAX_LOG_MESSAGE_LENGTH) instead of
+// O(input length) on multi-megabyte scraped payloads.
+const REDACT_SCAN_LENGTH = MAX_LOG_MESSAGE_LENGTH * 4;
+
+// Characters that can appear inside an unbroken secret token (hex, base64url,
+// JWT segments, API keys). Used to push the REDACT_SCAN_LENGTH cut point past
+// the end of whatever token it would otherwise land in the middle of.
+const TOKEN_CHAR_PATTERN = /[A-Za-z0-9_\-./+=]/;
+// Hard cap on how far the boundary can be pushed past REDACT_SCAN_LENGTH —
+// bounds the extra work to a fixed amount even against an adversarial run of
+// token-like characters.
+const REDACT_SCAN_EXTENSION_MAX = 2048;
+
 export function redactSecrets(message: string): string {
   // Redact BEFORE truncating. Every pattern below needs to see a secret in
   // full to match it (e.g. LONG_HEX_SECRET_PATTERN requires 32+ contiguous
@@ -140,7 +159,18 @@ export function redactSecrets(message: string): string {
   // whatever fragment survives the cut (e.g. the first 20 chars of a 64-hex
   // browser auth secret) too short for any pattern to recognize, so it was
   // written out in the clear right before the "…[truncated N chars]" marker.
-  let out = message;
+  const rawLength = message.length;
+  // The REDACT_SCAN_LENGTH cut below is itself a truncation, subject to the
+  // exact same mid-token risk — extend it forward past any run of token
+  // characters straddling the cut point so it never slices a secret in half.
+  let scanEnd = REDACT_SCAN_LENGTH;
+  if (rawLength > scanEnd) {
+    const extensionLimit = Math.min(rawLength, scanEnd + REDACT_SCAN_EXTENSION_MAX);
+    while (scanEnd < extensionLimit && TOKEN_CHAR_PATTERN.test(message.charAt(scanEnd))) {
+      scanEnd += 1;
+    }
+  }
+  let out = rawLength > scanEnd ? message.slice(0, scanEnd) : message;
   out = out.replace(ANSI_PATTERN, '');
   out = out.replace(URL_CREDENTIALS_PATTERN, '$1[REDACTED]@');
   // Redact whole-token formats BEFORE the key/value pass: the KV value class
@@ -172,7 +202,12 @@ export function redactSecrets(message: string): string {
     if (lastKeptUnit >= 0xd800 && lastKeptUnit <= 0xdbff) keep -= 1;
   }
   if (out.length > MAX_LOG_MESSAGE_LENGTH) {
-    out = `${out.slice(0, keep)}…[truncated ${out.length - keep} chars]`;
+    // rawLength can be SMALLER than out.length when redaction net-expands text
+    // (e.g. "key=1" → "key=[REDACTED]"), which would make `rawLength - keep`
+    // negative. out.length is always > keep in this branch, so the max() is
+    // never negative; it still prefers the larger rawLength when the
+    // REDACT_SCAN_LENGTH cut discarded raw content redaction then shrank.
+    out = `${out.slice(0, keep)}…[truncated ${Math.max(rawLength, out.length) - keep} chars]`;
   }
   return out;
 }

package/src/utils/text-utils.ts

@@ -188,8 +188,12 @@ export function formatDuration(ms: number): string {
   if (ms < 1000) return `${ms}ms`;
   const seconds = ms / 1000;
   if (seconds < 60) return `${seconds.toFixed(1)}s`;
-  const minutes = Math.floor(seconds / 60);
-  const remainingSeconds = Math.round(seconds % 60);
+  // Round the total first, then derive minutes/seconds from that integer —
+  // rounding remainingSeconds independently of minutes let e.g. 59.6s round up
+  // to a nonsensical "Xm 60s" instead of rolling over to the next minute.
+  const totalSeconds = Math.round(seconds);
+  const minutes = Math.floor(totalSeconds / 60);
+  const remainingSeconds = totalSeconds % 60;
   if (minutes < 60) return `${minutes}m ${remainingSeconds}s`;
   const hours = Math.floor(minutes / 60);
   const remainingMinutes = minutes % 60;

package/src/youtube/potoken.ts

@@ -42,6 +42,7 @@
  */
 
 import { logger } from '../logger.ts';
+import { safeUnref } from '../utils/safe-unref.ts';
 import { metrics } from '../utils/metrics.ts';
 
 /**
@@ -76,6 +77,22 @@ function describeError(error: unknown): string {
 // Module-level mutex: serialize all globalThis mutation across concurrent callers.
 let mintChain: Promise<unknown> = Promise.resolve();
 
+// Set when a hard-release fires while its doMint() is still bridging globals —
+// globalThis is a single process-global resource, so unlike a per-context lock
+// ANY new bridge while one is still installed is unsafe, not just one from the
+// same caller. Blocks new mints until the stuck call's own doMint() actually
+// settles (success or failure) and unbridges for real.
+let wedged = false;
+
+// How long the mutex waits for a straggling doMint() before force-releasing
+// the chain for the next caller. doMint() awaits non-fetch operations with no
+// abort signal (interpreter eval, BotGuard VM execution against a
+// hostile/malformed attestation script) that could in principle hang forever;
+// without this bound, a single wedged call would permanently break YouTube
+// transcript fetching for the rest of the process. Mirrors the hard-release
+// timer on the browser page-creation mutex in thread-worker-messaging.ts.
+const MINT_HARD_RELEASE_MS = 5 * 60 * 1000;
+
 /**
  * Mint a session-bound token (for `identifiers[0]`, the visitorData) plus one
  * content-bound token per remaining identifier (the video IDs), all from a
@@ -88,10 +105,39 @@ export async function mintPoTokens(
   identifiers: string[],
   opts: MintOptions = {},
 ): Promise<Map<string, string>> {
+  if (wedged) {
+    throw new Error('PoToken minter is wedged by a prior mint that never settled; refusing to start a new mint until it resolves.');
+  }
   // Chain on the mutex so only one bridged section runs at a time.
   const run = mintChain.then(() => doMint(identifiers, opts), () => doMint(identifiers, opts));
-  // Keep the chain alive regardless of this call's success/failure.
-  mintChain = run.then(() => undefined, () => undefined);
+  // Keep the chain alive regardless of this call's success/failure, but bound
+  // how long the NEXT caller waits on it: if `run` never settles, force the
+  // chain open after the hard timeout rather than wedging every future
+  // mintPoTokens() call behind it forever. Marking `wedged` (rather than
+  // letting the next caller straight through to its own doMint()) prevents
+  // two calls from bridging globalThis at once — the abandoned `run` may
+  // still be mid-bridge in the background, and a second bridge/unbridge pair
+  // racing it would restore the WRONG saved descriptors, permanently
+  // corrupting globalThis.window/document/navigator for the process.
+  //
+  // Derived from `settled` (which never rejects) rather than attaching
+  // directly to `run` — `run.finally(...)` would return a new, unconsumed
+  // promise that adopts `run`'s rejection, becoming an unhandled rejection
+  // whenever doMint() throws.
+  const settled = run.then(() => undefined, () => undefined);
+  // Once doMint() actually settles (success or failure), its own `finally`
+  // always runs unbridge() for real — safe for a new mint to bridge again.
+  settled.then(() => { wedged = false; });
+  let hardTimer: NodeJS.Timeout;
+  const hardRelease = new Promise<void>(resolve => {
+    hardTimer = setTimeout(() => {
+      logger.warn('[youtube] potoken mint mutex force-released — doMint() did not settle within the hard timeout');
+      wedged = true;
+      resolve();
+    }, MINT_HARD_RELEASE_MS);
+    safeUnref(hardTimer);
+  });
+  mintChain = Promise.race([settled, hardRelease]).finally(() => clearTimeout(hardTimer));
   return run;
 }
 

package/src/youtube/transcript-client.ts

@@ -20,6 +20,7 @@
 import { withTimeout, retryWithBackoff, createTimeoutSignal } from '../web-research/retry-utils.ts';
 import { CircuitBreaker } from '../utils/circuit-breaker.ts';
 import { logger } from '../logger.ts';
+import { redactSecrets } from '../utils/log-utils.ts';
 import { metrics } from '../utils/metrics.ts';
 import { readTextCapped } from '../utils/http-body.ts';
 import { mintPoTokens } from './potoken.ts';
@@ -355,8 +356,10 @@ function parseJson3(body: string): string {
     // Log the real cause: an unparseable body is NOT necessarily an empty
     // bot-protection response (the caller's default diagnosis) — it can be an HTML
     // interstitial or a response-shape change. A short snippet disambiguates without
-    // dumping a huge/again-sensitive body.
-    logger.debug(`[youtube] timedtext body was not valid json3 (first 120 chars): ${body.slice(0, 120)}`);
+    // dumping a huge/again-sensitive body. Redact BEFORE truncating to 120 chars —
+    // slicing first can cut a secret-shaped token mid-way, leaving a fragment too
+    // short for redactSecrets' whole-token patterns to recognize.
+    logger.debug(`[youtube] timedtext body was not valid json3 (first 120 chars): ${redactSecrets(body).slice(0, 120)}`);
     return '';
   }
   const events = (parsed as { events?: unknown[] }).events;