@lincoln504/pi-research 1.7.7 → 1.7.9

package/src/web-research/browser-search.ts

@@ -323,20 +323,18 @@ export async function performSearch(
 
         // A missing native addon is a BROKEN INSTALL, and the generic advice below
         // actively misleads: it sends the reader to check their network and their
-        // machine load when neither is involved and neither can be acted on. npm 12
-        // defaults `allowScripts` to off, so a plain install no longer builds
-        // camoufox-js's better-sqlite3 dependency and every browser worker dies
-        // identically. Say that instead, and say what fixes it.
+        // machine load when neither is involved and neither can be acted on. A
+        // dropped platform package (npm/cli#4828) or an install that skipped a native
+        // dependency's binding makes every browser worker die identically. Say that
+        // instead, and say what fixes it.
         if (isNativeBindingError(sampleWorkerError)) {
             metrics.increment('browser_search_total_failures_total', 1, { cause: 'native_binding' });
             throw new Error(
                 `Search completely failed: ${reason}, all with a missing native module. ` +
-                `This is an incomplete install, not a network problem: better-sqlite3 13 ` +
-                `ships prebuilt bindings for every supported platform and loads them at ` +
-                `runtime, so a missing binding means an old or broken npm install. ` +
-                `Upgrade npm to ≥12 (npm install -g npm@12) and reinstall this package — ` +
-                `modern npm skips the needless recompile that fails on toolchain-less ` +
-                `machines, and the prebuilt binding then loads as-is.` +
+                `This is an incomplete install, not a network problem: the browser launcher's ` +
+                `native dependency (impit) ships a prebuilt binding per platform as a separate ` +
+                `package, and npm sometimes drops it (npm/cli#4828). ` +
+                `Reinstall this package (npm install, or pi install again) to restore it.` +
                 (sampleWorkerError ? ` Last worker error: ${sampleWorkerError}` : '')
             );
         }

package/src/web-research/pdf-extract-worker.mjs

@@ -213,11 +213,15 @@ var COOKIE_HEADER_PATTERN = new RegExp(
   "gi"
 );
 var SENSITIVE_KV_PATTERN = new RegExp(
-  `${B_LEFT}(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([^\\s"',&)]+)`,
+  `${B_LEFT}(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|token|key)${B_RIGHT}(["']?\\s*[:=]\\s*)("[^"\\r\\n]{0,1024}"|'[^'\\r\\n]{0,1024}'|["']?[^\\s"',&)]+)`,
+  "gi"
+);
+var AUTH_SCHEME_PATTERN = new RegExp(
+  `${B_LEFT}(authorization)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([A-Za-z][A-Za-z0-9_-]{0,31})[ \\t]+[^\\s"',&)]+`,
   "gi"
 );
 var KNOWN_TOKEN_PATTERN = new RegExp(
-  `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
+  `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35}|npm_[A-Za-z0-9]{36}|hf_[A-Za-z0-9]{30,}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
   "g"
 );
 var JWT_PATTERN = new RegExp(
@@ -251,7 +255,12 @@ function redactSecrets(message) {
   out = out.replace(BASIC_AUTH_PATTERN, "Basic [REDACTED]");
   out = out.replace(BEARER_AUTH_PATTERN, "Bearer [REDACTED]");
   out = out.replace(COOKIE_HEADER_PATTERN, (_m, key, sep) => `${key}${sep}[REDACTED]`);
-  out = out.replace(SENSITIVE_KV_PATTERN, (_m, key, sep) => `${key}${sep}[REDACTED]`);
+  out = out.replace(AUTH_SCHEME_PATTERN, (_m, key, sep, scheme) => `${key}${sep}${scheme} [REDACTED]`);
+  out = out.replace(SENSITIVE_KV_PATTERN, (_m, key, sep, val) => {
+    const q = val[0] === '"' || val[0] === "'" ? val[0] : "";
+    const closed = q !== "" && val.length > 1 && val.endsWith(q);
+    return `${key}${sep}${q}[REDACTED]${closed ? q : ""}`;
+  });
   out = out.replace(KNOWN_TOKEN_PATTERN, "[REDACTED]");
   out = out.replace(PROVIDER_TOKEN_PATTERN, "[REDACTED]");
   out = out.replace(LONG_HEX_SECRET_PATTERN, "[REDACTED]");

package/src/web-research/scraper-utils.ts

@@ -382,18 +382,27 @@ export function formatErrorWithCause(err: unknown): string {
 function isPrivateIp(ip: string): boolean {
   const parts = ip.split('.').map(Number);
   if (parts.length !== 4 || parts.some(p => isNaN(p))) return false;
-  const [a, b] = parts;
+  const [a, b, c] = parts;
   if (a === undefined || b === undefined) return false;
   // 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10 (RFC 6598 CGNAT / shared address space,
   // routinely routed to internal infra by cloud/hosting providers), 127.0.0.0/8,
   // 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16, 224.0.0.0/4 (multicast),
-  // 240.0.0.0/4 (reserved)
+  // 240.0.0.0/4 (reserved).
+  // Also blocked: 198.18.0.0/15 (RFC 2544 benchmarking, sometimes routed by
+  // broken tooling), 192.0.0.0/24 (RFC 6890 IETF protocol assignments) and the
+  // three documentation ranges 192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24
+  // (TEST-NET-1/2/3, RFC 5737). None is a legitimate scrape target, and a
+  // mis-issued NAT or a hostile page's SSRF probe can name them.
   return (
     a === 0 || a === 10 || a === 127 ||
     (a === 100 && b >= 64 && b <= 127) ||
     (a === 169 && b === 254) ||
     (a === 172 && b >= 16 && b <= 31) ||
     (a === 192 && b === 168) ||
+    (a === 198 && (b === 18 || b === 19)) ||
+    (a === 192 && b === 0 && (c === 0 || c === 2)) ||
+    (a === 198 && b === 51 && c === 100) ||
+    (a === 203 && b === 0 && c === 113) ||
     a >= 224
   );
 }
@@ -551,20 +560,43 @@ export function trimFrontMatter(markdown: string): string {
  * leaves extra blank lines, so they are collapsed again.
  */
 export function cleanConvertedMarkdown(markdown: string): string {
-  return stripEmptyFragmentLinks(stripImageLinks(trimFrontMatter(markdown)))
-    .replace(/\n{3,}/g, '\n\n')
-    .trim();
+  return collapseBlankLines(stripEmptyFragmentLinks(stripImageLinks(trimFrontMatter(markdown)))).trim();
+}
+
+/** Opening/closing fence of a fenced code block (CommonMark: up to 3 spaces, 3+ backticks or tildes). */
+const CODE_FENCE = /^ {0,3}(`{3,}|~{3,})/;
+
+/**
+ * Collapse runs of blank lines to one, EXCEPT inside fenced code blocks, where
+ * blank lines are content (PEP 8 spacing, Makefiles, ASCII art). An unclosed
+ * fence runs to the end of the document, as in CommonMark.
+ */
+export function collapseBlankLines(markdown: string): string {
+  if (!markdown.includes('\n\n\n')) return markdown;
+  const out: string[] = [];
+  let fence: string | null = null;
+  for (const line of markdown.split('\n')) {
+    const m = CODE_FENCE.exec(line);
+    if (fence === null) {
+      if (m) fence = m[1]!;
+    } else if (m && m[1]![0] === fence[0] && m[1]!.length >= fence.length && line.slice(m[0].length).trim() === '') {
+      fence = null;
+    }
+    if (fence === null && line === '' && out.length > 0 && out[out.length - 1] === '') continue;
+    out.push(line);
+  }
+  return out.join('\n');
 }
 
 /**
  * Strip image links from markdown
  */
 export function stripImageLinks(markdown: string): string {
-  return markdown
-    .replace(MARKDOWN_IMAGE_PATTERN, '')
-    .replace(IMAGE_LINK_PATTERN, '$1')
-    .replace(/\n{3,}/g, '\n\n')
-    .trim();
+  return collapseBlankLines(
+    markdown
+      .replace(MARKDOWN_IMAGE_PATTERN, '')
+      .replace(IMAGE_LINK_PATTERN, '$1'),
+  ).trim();
 }
 
 /**

package/src/web-research/web-scraper.ts

@@ -41,7 +41,7 @@ let playwrightAvailable: boolean = false;
 let markdownConverterPromise: Promise<(html: string) => Promise<string>> | null = null;
 
 export function initScraperDependencies(): void {
-  playwrightAvailable = checkModule('playwright-core') && checkModule('camoufox-js');
+  playwrightAvailable = checkModule('playwright-core') && checkModule('@camoufox/camoufox');
 }
 initScraperDependencies();