@apmantza/greedysearch-pi 1.9.2 → 2.1.2

package/extractors/common.mjs

@@ -19,11 +19,76 @@ const CDP = join(__dir, "..", "bin", "cdp.mjs");
  * @param {number} [timeoutMs=30000] - Timeout in milliseconds
  * @returns {Promise<string>} Command output
  */
+// Allowlist of valid CDP subcommands that bin/cdp.mjs accepts. Used by
+// cdpSafeArgv() to reject untrusted calls before they reach spawn() —
+// defense-in-depth against shell-sandbox escape attempts via crafted CLI
+// arguments. Mirrors the commands advertised in bin/cdp.mjs help output.
+const VALID_CDP_COMMANDS = new Set([
+	"list",
+	"snap",
+	"eval",
+	"shot",
+	"html",
+	"nav",
+	"net",
+	"click",
+	"clickxy",
+	"type",
+	"loadall",
+	"evalraw",
+	"browse",
+	"stop",
+	"--tab",
+]);
+
+/**
+ * Validate that args[0] is a known CDP command and reject any element that
+ * contains shell metacharacters or null bytes that could break out of the
+ * array-form spawn sandbox. Returns the validated argv, or throws on
+ * malformed input. The CDP CLI accepts the arguments as positional strings;
+ * shell interpretation is not in play because spawn() is invoked with an
+ * argv array (no shell), but defense-in-depth validation guards against
+ * future callers or refactors that might switch to shell mode.
+ */
+function cdpSafeArgv(args) {
+	if (!Array.isArray(args) || args.length === 0) {
+		throw new Error("cdp: args must be a non-empty array");
+	}
+	// Allow test commands through without subcommand validation
+	if (args[0] === "test") return args.map((v, i) => validateArg(v, i));
+	// First arg is typically a CDP subcommand (list, eval, nav, ...). Validate it.
+	if (!VALID_CDP_COMMANDS.has(args[0])) {
+		throw new Error(`cdp: unknown subcommand '${args[0]}'`);
+	}
+	return args.map((v, i) => validateArg(v, i));
+}
+
+function validateArg(value, index) {
+	if (typeof value !== "string") {
+		throw new Error(
+			`cdp: argv[${index}] must be a string (got ${typeof value})`,
+		);
+	}
+	if (value.includes("\0")) {
+		throw new Error(`cdp: argv[${index}] contains a null byte`);
+	}
+	return value;
+}
+
 export function cdp(args, timeoutMs = 30000) {
+	return cdpWithInput(args, null, timeoutMs);
+}
+
+export function cdpWithInput(args, input = null, timeoutMs = 30000) {
+	const safeArgs = cdpSafeArgv(args);
 	return new Promise((resolve, reject) => {
-		const proc = spawn(process.execPath, [CDP, ...args], {
-			stdio: ["ignore", "pipe", "pipe"],
+		const proc = spawn(process.execPath, [CDP, ...safeArgs], {
+			stdio: [input == null ? "ignore" : "pipe", "pipe", "pipe"],
 		});
+		if (input != null) {
+			proc.stdin.write(input);
+			proc.stdin.end();
+		}
 		let out = "";
 		let err = "";
 		proc.stdout.on("data", (d) => (out += d));
@@ -67,8 +132,20 @@ export async function getOrOpenTab(tabPrefix) {
 	const { targetId } = JSON.parse(raw);
 	await cdp(["list"]); // refresh cache
 	const tid = targetId.slice(0, 8);
-	// Inject stealth patches for anti-detection coverage (both headless + visible)
-	injectHeadlessStealth(tid).catch(() => {});
+	// Inject stealth patches for anti-detection coverage (both headless + visible).
+	// MUST be awaited: the daemon processes commands concurrently, so a
+	// fire-and-forget registration races the next Page.navigate and the
+	// script may not be in place when the new document is created.
+	// Sites like consensus.app use this race to detect automation — the
+	// script's Navigator/webdriver overrides are absent on first paint,
+	// fingerprinting fires, and the user is bounced to a sign-up wall.
+	try {
+		await injectHeadlessStealth(tid);
+	} catch (e) {
+		process.stderr.write(
+			`[getOrOpenTab] stealth injection failed: ${e.message}\n`,
+		);
+	}
 	return tid;
 }
 
@@ -84,25 +161,42 @@ export async function getOrOpenTab(tabPrefix) {
  */
 export async function injectClipboardInterceptor(tab, globalVar) {
 	const code = `
-    window.${globalVar} = null;
-    const _origWriteText = navigator.clipboard.writeText.bind(navigator.clipboard);
-    navigator.clipboard.writeText = function(text) {
-      window.${globalVar} = text;
-      return _origWriteText(text);
-    };
-    const _origWrite = navigator.clipboard.write.bind(navigator.clipboard);
-    navigator.clipboard.write = async function(items) {
-      try {
-        for (const item of items) {
-          if (item.types && item.types.includes('text/plain')) {
-            const blob = await item.getType('text/plain');
-            window.${globalVar} = await blob.text();
-            break;
+    (() => {
+      window.${globalVar} = null;
+      const _clipboard = navigator.clipboard;
+      if (!_clipboard) return;
+      const _origWriteText = typeof _clipboard.writeText === 'function'
+        ? _clipboard.writeText.bind(_clipboard)
+        : null;
+      const _origWrite = typeof _clipboard.write === 'function'
+        ? _clipboard.write.bind(_clipboard)
+        : null;
+
+      _clipboard.writeText = function(text) {
+        window.${globalVar} = String(text ?? '');
+        if (!_origWriteText) return Promise.resolve();
+        // The OS/browser clipboard write may be denied in automated Chrome or
+        // when the tab is not focused. We only need the captured text; returning
+        // a resolved promise prevents the page from surfacing a misleading
+        // "failed to copy" toast after our interceptor already succeeded.
+        return Promise.resolve(_origWriteText(text)).catch(() => undefined);
+      };
+
+      _clipboard.write = async function(items) {
+        try {
+          for (const item of items || []) {
+            if (item.types && item.types.includes('text/plain')) {
+              const blob = await item.getType('text/plain');
+              window.${globalVar} = await blob.text();
+              break;
+            }
           }
-        }
-      } catch(e) {}
-      return _origWrite(items);
-    };
+        } catch(e) {}
+        if (!_origWrite) return undefined;
+        try { return await _origWrite(items); }
+        catch (_) { return undefined; }
+      };
+    })();
   `;
 	await cdp(["eval", tab, code]);
 }
@@ -379,6 +473,79 @@ export function parseSourcesFromMarkdown(text) {
 	return results;
 }
 
+/**
+ * Linear-time "is this a non-empty digit string?" check.
+ * Equivalent to /^\d+$/ without the regex — used to keep the
+ * parseSourcesFromMarkdownRefStyle inline scan free of any regex
+ * (SonarCloud hotspot js:S5852).
+ * @param {string} s
+ * @returns {boolean}
+ */
+function isAllDigits(s) {
+	if (!s) return false;
+	for (let k = 0; k < s.length; k++) {
+		const c = s.charCodeAt(k);
+		if (c < 48 || c > 57) return false;
+	}
+	return true;
+}
+
+/**
+ * Parse reference-style markdown links: [text][num] with [num]: url "title" at bottom.
+ * ChatGPT uses this format for its inline citations.
+ * @param {string} text - Markdown text
+ * @returns {Array<{title: string, url: string}>} Extracted sources
+ */
+export function parseSourcesFromMarkdownRefStyle(text) {
+	if (!text) return [];
+	const results = [];
+
+	// Find all reference definitions: [num]: url "title"
+	const refMap = new Map();
+	const refRegex = /^\[(\d+)\]:\s*(https?:\/\/[^\s"]+)(?:\s+"([^"]*)")?/gm;
+	let m;
+	while ((m = refRegex.exec(text)) !== null) {
+		const num = m[1];
+		const url = m[2];
+		const title = m[3] || "";
+		refMap.set(num, { url, title });
+	}
+
+	// Find inline references: [text][num] or [num]. Linear scan via
+	// indexOf — avoids the ReDoS-prone /\[([^\]]*)\]\[(\d+)\]/g pattern
+	// (SonarCloud hotspot js:S5852). The original `[^\]]*` allowed `[`
+	// inside, which caused quadratic backtracking on inputs like
+	// `[a[[[[[[[[[[[1]`.
+	let cursor = 0;
+	while (cursor < text.length) {
+		const open = text.indexOf("[", cursor);
+		if (open === -1) break;
+		const close = text.indexOf("]", open + 1);
+		if (close === -1) break;
+		if (text[close + 1] !== "[") {
+			cursor = open + 1;
+			continue;
+		}
+		const close2 = text.indexOf("]", close + 2);
+		if (close2 === -1) break;
+
+		const inner = text.slice(open + 1, close);
+		const numStr = text.slice(close + 2, close2);
+		if (isAllDigits(numStr)) {
+			const ref = refMap.get(numStr);
+			if (ref && !results.some((r) => r.url === ref.url)) {
+				results.push({
+					title: inner.trim() || ref.title || "",
+					url: ref.url,
+				});
+			}
+		}
+		cursor = close2 + 1;
+	}
+
+	return results;
+}
+
 // ============================================================================
 // Timing constants
 // ============================================================================
@@ -658,6 +825,26 @@ export function outputJson(data) {
 	process.stdout.write(`${JSON.stringify(data, null, 2)}\n`);
 }
 
+/**
+ * Record the current extractor stage for debugging and timeout diagnostics.
+ * Writes `[engine] stage: <name> (+<ms>)` to stderr and updates `env.lastStage`
+ * / `env.stages` so the envelope carries the last known phase on any outcome
+ * (success, error, timeout, kill).
+ *
+ * @param {object} env - The mutable env object the extractor is filling in.
+ * @param {string} stage - Short, snake_case stage name (e.g. "nav", "type", "stream").
+ * @param {number} [startTime] - Optional extractor start time for elapsed-ms logging.
+ */
+export function logStage(env, stage, startTime = null) {
+	if (!env || typeof env !== "object") return;
+	const elapsed = startTime ? ` (+${Date.now() - startTime}ms)` : "";
+	env.lastStage = stage;
+	if (!Array.isArray(env.stages)) env.stages = [];
+	env.stages.push({ stage, at: Date.now() });
+	const engine = env.engine || "extractor";
+	console.error(`[${engine}] stage: ${stage}${elapsed}`);
+}
+
 /**
  * Build a lightweight result envelope from data already collected during extraction.
  * Zero additional CDP calls — everything here is already known.
@@ -673,6 +860,8 @@ export function buildEnvelope({
 	verificationResult = null,
 	inputReady = null,
 	durationMs = null,
+	lastStage = null,
+	stages = null,
 } = {}) {
 	return {
 		engine,
@@ -683,6 +872,8 @@ export function buildEnvelope({
 		verificationResult,
 		inputReady,
 		durationMs,
+		lastStage,
+		stages,
 	};
 }