@apmantza/greedysearch-pi 1.9.2 → 2.0.0

package/extractors/chatgpt.mjs

@@ -0,0 +1,436 @@
+#!/usr/bin/env node
+
+// extractors/chatgpt.mjs
+// Navigate chatgpt.com, submit query, wait for answer, extract answer + sources.
+//
+// Usage:
+//   node extractors/chatgpt.mjs "<query>" [--tab <prefix>]
+//
+// Output (stdout): JSON { answer, sources, query, url }
+// Errors go to stderr only — stdout is always clean JSON for piping.
+
+import {
+	buildEnvelope,
+	cdp,
+	cdpWithInput,
+	formatAnswer,
+	getOrOpenTab,
+	handleError,
+	injectClipboardInterceptor,
+	jitter,
+	logStage,
+	outputJson,
+	parseArgs,
+	parseSourcesFromMarkdown,
+	parseSourcesFromMarkdownRefStyle,
+	prepareArgs,
+	validateQuery,
+	waitForSelector,
+	waitForStreamComplete,
+} from "./common.mjs";
+import { dismissConsent, handleVerification } from "./consent.mjs";
+
+const GLOBAL_VAR = "__chatgptClipboard";
+const PROSE_SELECTOR = "div.ProseMirror";
+const SEND_SELECTOR = 'button[data-testid="send-button"]';
+const COPY_SELECTOR = 'button[data-testid="copy-turn-action-button"]';
+
+// ============================================================================
+// ChatGPT-specific helpers
+// ============================================================================
+
+async function typeAndSubmit(tab, query) {
+	// Focus the ProseMirror editor
+	await cdp(["click", tab, PROSE_SELECTOR]);
+	await new Promise((r) => setTimeout(r, jitter(200)));
+
+	// Type via CDP (sends Input.insertText). Use stdin so long synthesis
+	// prompts do not hit Windows command-line length limits.
+	await cdpWithInput(["type", tab, "--stdin"], query);
+	await new Promise((r) => setTimeout(r, jitter(300)));
+
+	// Click send button
+	const sendCode = `
+		(() => {
+			const btn = document.querySelector('${SEND_SELECTOR}');
+			if (!btn) return 'no-send';
+			btn.click();
+			return 'ok';
+		})()
+	`;
+	const sendResult = await cdp(["eval", tab, sendCode]);
+	if (sendResult === "no-send")
+		throw new Error("ChatGPT send button not found");
+	await new Promise((r) => setTimeout(r, jitter(300)));
+}
+
+/**
+ * Inline selector for waitForStreamComplete: returns the assistant message
+ * that comes AFTER the last user message, or null if none exists. This
+ * skips chatgpt.com's static pre-rendered greeting card (which is
+ * `data-turn-start-message="true"` and lives on the homepage before any
+ * conversation) so short answers like "Hello! 👋" don't get confused with
+ * the 32-char placeholder.
+ */
+const CHATGPT_RESPONSE_SELECTOR = String.raw`(() => {
+	const all = document.querySelectorAll('[data-message-author-role]');
+	let lastUserIdx = -1;
+	for (let i = 0; i < all.length; i++) {
+		if (all[i].getAttribute('data-message-author-role') === 'user') lastUserIdx = i;
+	}
+	if (lastUserIdx < 0) return null;
+	let bestEl = null;
+	let bestLen = 0;
+	for (let i = lastUserIdx + 1; i < all.length; i++) {
+		if (all[i].getAttribute('data-message-author-role') === 'assistant') {
+			const len = (all[i].innerText || '').length;
+			if (len > bestLen) { bestLen = len; bestEl = all[i]; }
+		}
+	}
+	return bestEl;
+})()`;
+
+/**
+ * Wait for ChatGPT's response to finish streaming. Delegates to the shared
+ * waitForStreamComplete in common.mjs with a custom selector that skips the
+ * static homepage greeting card. minLength: 1 means any non-empty response
+ * is considered "started" — short answers like "Hello! 👋" (8 chars) used
+ * to burn the full 65s budget under the old 50-char threshold.
+ */
+async function waitForResponse(tab, timeoutMs = 20000) {
+	return waitForStreamComplete(tab, {
+		timeout: timeoutMs,
+		interval: 600,
+		stableRounds: 3,
+		minLength: 1,
+		selector: CHATGPT_RESPONSE_SELECTOR,
+	});
+}
+
+/**
+ * Node-side fallback for chatgpt stream completion. Used when the in-browser
+ * poll times out (typically because Chrome throttles background tabs to 1Hz
+ * when 3+ extractors run in parallel in `all` mode). Polls the same
+ * greeting-card-skipping selector via short independent Runtime.evaluate
+ * calls so the WebSocket is free between polls.
+ */
+async function pollForResponseNodeSide(tab, maxMs = 15000) {
+	const deadline = Date.now() + maxMs;
+	let lastLen = 0;
+	let stableRounds = 0;
+	while (Date.now() < deadline) {
+		const result = await cdp(
+			["eval", tab, `${CHATGPT_RESPONSE_SELECTOR}?.innerText?.length ?? 0`],
+			4000,
+		).catch(() => "0");
+		const len = parseInt(result, 10) || 0;
+		if (len >= 1 && len === lastLen) {
+			stableRounds++;
+			if (stableRounds >= 3) return len;
+		} else {
+			lastLen = len;
+			stableRounds = 0;
+		}
+		await new Promise((r) => setTimeout(r, 1200));
+	}
+	return lastLen;
+}
+
+async function extractAnswerFromDom(tab) {
+	const raw = await cdp([
+		"eval",
+		tab,
+		String.raw`
+		(() => {
+			// Find the assistant message that comes AFTER the last user message,
+			// not the absolute last assistant element. The chatgpt.com homepage
+			// has a static pre-rendered greeting card that renders as a
+			// [data-message-author-role="assistant"] element with
+			// data-turn-start-message="true" — it must be skipped or the
+			// static "Hello! How can I help you today?" placeholder gets
+			// returned as the answer to a query the assistant never answered.
+			const all = Array.from(document.querySelectorAll('[data-message-author-role]'));
+			let lastUserIdx = -1;
+			for (let i = 0; i < all.length; i++) {
+				if (all[i].getAttribute('data-message-author-role') === 'user') {
+					lastUserIdx = i;
+				}
+			}
+			if (lastUserIdx < 0) {
+				// No user message at all — page is still on the homepage.
+				return JSON.stringify({
+					answer: '',
+					sources: [],
+					skipped: 'no-user-message',
+				});
+			}
+			let assistant = null;
+			for (let i = lastUserIdx + 1; i < all.length; i++) {
+				if (all[i].getAttribute('data-message-author-role') === 'assistant') {
+					assistant = all[i];
+				}
+			}
+			if (!assistant) {
+				return JSON.stringify({
+					answer: '',
+					sources: [],
+					skipped: 'no-assistant-response',
+				});
+			}
+			const answer = (assistant.innerText || assistant.textContent || '').trim();
+			const seen = new Set();
+			const sources = [];
+			for (const link of assistant.querySelectorAll('a[href]')) {
+				const url = link.href;
+				if (!url || seen.has(url)) continue;
+				seen.add(url);
+				const title = (link.innerText || link.textContent || '').replace(/\s+/g, ' ').trim();
+				sources.push({ title, url });
+				if (sources.length >= 10) break;
+			}
+			return JSON.stringify({ answer, sources });
+		})()
+	`,
+	]);
+	try {
+		return JSON.parse(raw);
+	} catch {
+		return { answer: "", sources: [], skipped: "parse-error" };
+	}
+}
+
+async function extractAnswer(tab, env) {
+	// Click the copy button on the assistant's response (after the last
+	// user message). The old `buttons[buttons.length - 1]` picked the
+	// absolute last copy button on the page — which is the USER message's
+	// copy button when the assistant response is still empty (0 chars) and
+	// has no copy button of its own. That copied the user's query into
+	// the clipboard interceptor and returned it as the "answer".
+	//
+	// If the assistant message has no copy button yet (still streaming, or
+	// the React tree hasn't rendered the button after streaming completed),
+	// we deliberately click NOTHING rather than falling back to the last
+	// copy button on the page. An empty clipboard routes us to the DOM
+	// fallback, which correctly targets the assistant message after the
+	// last user message and returns its innerText.
+	await cdp([
+		"eval",
+		tab,
+		`(() => {
+			const all = document.querySelectorAll('[data-message-author-role]');
+			let lastUserIdx = -1;
+			for (let i = 0; i < all.length; i++) {
+				if (all[i].getAttribute('data-message-author-role') === 'user') lastUserIdx = i;
+			}
+			if (lastUserIdx < 0) return 'no-user';
+			let assistantCopy = null;
+			for (let i = lastUserIdx + 1; i < all.length; i++) {
+				if (all[i].getAttribute('data-message-author-role') === 'assistant') {
+					const btn = all[i].querySelector('${COPY_SELECTOR}');
+					if (btn) assistantCopy = btn;
+				}
+			}
+			if (assistantCopy) { assistantCopy.click(); return 'clicked'; }
+			return 'no-assistant-copy';
+		})()`,
+	]);
+	await new Promise((r) => setTimeout(r, 600));
+
+	let answer = await cdp(["eval", tab, `window.${GLOBAL_VAR} || ''`]);
+	env.clipboardEmpty = !answer;
+
+	// Retry once if clipboard is empty — the assistant message may have
+	// finished streaming and the copy button may have rendered in the
+	// meantime.
+	if (!answer) {
+		console.error("[chatgpt] Clipboard empty, retrying in 2s...");
+		await cdp([
+			"eval",
+			tab,
+			`(() => {
+				const all = document.querySelectorAll('[data-message-author-role]');
+				let lastUserIdx = -1;
+				for (let i = 0; i < all.length; i++) {
+					if (all[i].getAttribute('data-message-author-role') === 'user') lastUserIdx = i;
+				}
+				if (lastUserIdx < 0) return 'no-user';
+				let assistantCopy = null;
+				for (let i = lastUserIdx + 1; i < all.length; i++) {
+					if (all[i].getAttribute('data-message-author-role') === 'assistant') {
+						const btn = all[i].querySelector('${COPY_SELECTOR}');
+						if (btn) assistantCopy = btn;
+					}
+				}
+				if (assistantCopy) { assistantCopy.click(); return 'clicked'; }
+				return 'no-assistant-copy';
+			})()`,
+		]);
+		await new Promise((r) => setTimeout(r, 2000));
+		answer = await cdp(["eval", tab, `window.${GLOBAL_VAR} || ''`]);
+		env.clipboardEmpty = !answer;
+	}
+
+	let domFallback = null;
+	if (!answer) {
+		domFallback = await extractAnswerFromDom(tab);
+		answer = domFallback.answer;
+		env.fallbackUsed = answer ? "dom" : null;
+	}
+
+	if (!answer) throw new Error("Clipboard interceptor returned empty text");
+
+	// Parse sources from both inline/reference-style markdown links and DOM links
+	// (DOM fallback preserves sources even when native clipboard copy fails).
+	const sourcesInline = parseSourcesFromMarkdown(answer);
+	const sourcesRef = parseSourcesFromMarkdownRefStyle(answer);
+	const sourceMap = new Map();
+	for (const s of [
+		...(domFallback?.sources || []),
+		...sourcesRef,
+		...sourcesInline,
+	]) {
+		if (s?.url && !sourceMap.has(s.url)) sourceMap.set(s.url, s);
+	}
+	const sources = Array.from(sourceMap.values()).slice(0, 10);
+
+	return { answer: answer.trim(), sources };
+}
+
+// ============================================================================
+// Main
+// ============================================================================
+
+const USAGE = 'Usage: node extractors/chatgpt.mjs "<query>" [--tab <prefix>]\n';
+
+async function main() {
+	const args = await prepareArgs(process.argv.slice(2));
+	validateQuery(args, USAGE);
+
+	const { query, tabPrefix, short } = parseArgs(args);
+	const startTime = Date.now();
+	const mode =
+		process.env.GREEDY_SEARCH_VISIBLE === "1" ? "visible" : "headless";
+
+	const env = {
+		engine: "chatgpt",
+		mode,
+		clipboardEmpty: null,
+		fallbackUsed: null,
+		blockedBy: null,
+		verificationResult: null,
+		inputReady: null,
+	};
+
+	try {
+		if (!tabPrefix) await cdp(["list"]);
+		const tab = await getOrOpenTab(tabPrefix);
+
+		const currentUrl = await cdp(["eval", tab, "document.location.href"]).catch(
+			() => "",
+		);
+		let onChatGPT = false;
+		try {
+			onChatGPT = new URL(currentUrl).hostname.toLowerCase() === "chatgpt.com";
+		} catch {}
+
+		if (!onChatGPT) {
+			logStage(env, "nav", startTime);
+			await cdp(["nav", tab, "https://chatgpt.com"], 20000);
+			await new Promise((r) => setTimeout(r, 600));
+		}
+		logStage(env, "consent", startTime);
+		await dismissConsent(tab, cdp);
+		logStage(env, "verification", startTime);
+		await handleVerification(tab, cdp, 10000);
+
+		logStage(env, "input-wait", startTime);
+		const inputReady = await waitForSelector(tab, PROSE_SELECTOR, 8000, 400);
+		env.inputReady = inputReady;
+		if (!inputReady) {
+			const bodyText = await cdp([
+				"eval",
+				tab,
+				`document.body?.innerText || ''`,
+			]).catch(() => "");
+			if (
+				/sign in|log in|sign up|\u03a3\u03cd\u03bd\u03b4\u03b5\u03c3\u03b7|login/i.test(
+					bodyText,
+				)
+			) {
+				throw new Error(
+					"ChatGPT requires sign-in — please sign in in the visible browser window",
+				);
+			}
+			throw new Error(
+				"ChatGPT input not found — page may be blocked or in unexpected state",
+			);
+		}
+
+		logStage(env, "clipboard-inject", startTime);
+		await injectClipboardInterceptor(tab, GLOBAL_VAR);
+		logStage(env, "type-and-submit", startTime);
+		await typeAndSubmit(tab, query);
+
+		logStage(env, "stream-wait", startTime);
+		// waitForStreamComplete handles the in-browser poll in a single
+		// Runtime.evaluate call. If the response is still streaming past
+		// 20s (slow under tab throttling in `all` mode), fall back to
+		// node-side polls that release the WebSocket between each call.
+		// Together they stay well within the engine's 80s outer budget.
+		let asstLen = 0;
+		try {
+			asstLen = await waitForResponse(tab, 20000);
+		} catch (e) {
+			logStage(env, "stream-poll-fallback", startTime);
+			asstLen = await pollForResponseNodeSide(tab, 15000);
+		}
+		env.assistantTextLen = asstLen;
+		if (asstLen < 1) {
+			console.error(
+				"[chatgpt] Warning: assistant response may not have completed",
+			);
+		}
+
+		logStage(env, "extract", startTime);
+		const { answer, sources, skipped } = await extractAnswer(tab, env);
+		// If the DOM fallback skipped the response (no real assistant
+		// message after the user's query), surface a clear error so the
+		// caller doesn't silently consume the static homepage greeting
+		// card as a real answer. The static card lives on chatgpt.com
+		// before any conversation; without this guard the extractor used
+		// to return "Hello! How can I help you today?" as a successful
+		// response to every query.
+		if (!answer) {
+			env.blockedBy = "no-response";
+			env.skipped = skipped || null;
+			throw new Error(
+				skipped === "no-user-message"
+					? "ChatGPT still on homepage — query was not submitted"
+					: skipped === "no-assistant-response"
+						? "ChatGPT did not return an assistant response after submit"
+						: "ChatGPT returned no answer — assistant never responded",
+			);
+		}
+		logStage(env, "done", startTime);
+
+		const finalUrl = await cdp(["eval", tab, "document.location.href"]).catch(
+			() => "https://chatgpt.com",
+		);
+		env.durationMs = Date.now() - startTime;
+		outputJson({
+			query,
+			url: finalUrl,
+			answer: formatAnswer(answer, short),
+			sources,
+			_envelope: buildEnvelope(env),
+		});
+	} catch (e) {
+		env.durationMs = Date.now() - startTime;
+		console.error(
+			`[chatgpt] error during stage '${env.lastStage || "unknown"}': ${e.message}`,
+		);
+		handleError(e, buildEnvelope(env));
+	}
+}
+
+main();

package/extractors/common.mjs

@@ -20,10 +20,18 @@ const CDP = join(__dir, "..", "bin", "cdp.mjs");
  * @returns {Promise<string>} Command output
  */
 export function cdp(args, timeoutMs = 30000) {
+	return cdpWithInput(args, null, timeoutMs);
+}
+
+export function cdpWithInput(args, input = null, timeoutMs = 30000) {
 	return new Promise((resolve, reject) => {
 		const proc = spawn(process.execPath, [CDP, ...args], {
-			stdio: ["ignore", "pipe", "pipe"],
+			stdio: [input == null ? "ignore" : "pipe", "pipe", "pipe"],
 		});
+		if (input != null) {
+			proc.stdin.write(input);
+			proc.stdin.end();
+		}
 		let out = "";
 		let err = "";
 		proc.stdout.on("data", (d) => (out += d));
@@ -67,8 +75,20 @@ export async function getOrOpenTab(tabPrefix) {
 	const { targetId } = JSON.parse(raw);
 	await cdp(["list"]); // refresh cache
 	const tid = targetId.slice(0, 8);
-	// Inject stealth patches for anti-detection coverage (both headless + visible)
-	injectHeadlessStealth(tid).catch(() => {});
+	// Inject stealth patches for anti-detection coverage (both headless + visible).
+	// MUST be awaited: the daemon processes commands concurrently, so a
+	// fire-and-forget registration races the next Page.navigate and the
+	// script may not be in place when the new document is created.
+	// Sites like consensus.app use this race to detect automation — the
+	// script's Navigator/webdriver overrides are absent on first paint,
+	// fingerprinting fires, and the user is bounced to a sign-up wall.
+	try {
+		await injectHeadlessStealth(tid);
+	} catch (e) {
+		process.stderr.write(
+			`[getOrOpenTab] stealth injection failed: ${e.message}\n`,
+		);
+	}
 	return tid;
 }
 
@@ -84,25 +104,42 @@ export async function getOrOpenTab(tabPrefix) {
  */
 export async function injectClipboardInterceptor(tab, globalVar) {
 	const code = `
-    window.${globalVar} = null;
-    const _origWriteText = navigator.clipboard.writeText.bind(navigator.clipboard);
-    navigator.clipboard.writeText = function(text) {
-      window.${globalVar} = text;
-      return _origWriteText(text);
-    };
-    const _origWrite = navigator.clipboard.write.bind(navigator.clipboard);
-    navigator.clipboard.write = async function(items) {
-      try {
-        for (const item of items) {
-          if (item.types && item.types.includes('text/plain')) {
-            const blob = await item.getType('text/plain');
-            window.${globalVar} = await blob.text();
-            break;
+    (() => {
+      window.${globalVar} = null;
+      const _clipboard = navigator.clipboard;
+      if (!_clipboard) return;
+      const _origWriteText = typeof _clipboard.writeText === 'function'
+        ? _clipboard.writeText.bind(_clipboard)
+        : null;
+      const _origWrite = typeof _clipboard.write === 'function'
+        ? _clipboard.write.bind(_clipboard)
+        : null;
+
+      _clipboard.writeText = function(text) {
+        window.${globalVar} = String(text ?? '');
+        if (!_origWriteText) return Promise.resolve();
+        // The OS/browser clipboard write may be denied in automated Chrome or
+        // when the tab is not focused. We only need the captured text; returning
+        // a resolved promise prevents the page from surfacing a misleading
+        // "failed to copy" toast after our interceptor already succeeded.
+        return Promise.resolve(_origWriteText(text)).catch(() => undefined);
+      };
+
+      _clipboard.write = async function(items) {
+        try {
+          for (const item of items || []) {
+            if (item.types && item.types.includes('text/plain')) {
+              const blob = await item.getType('text/plain');
+              window.${globalVar} = await blob.text();
+              break;
+            }
           }
-        }
-      } catch(e) {}
-      return _origWrite(items);
-    };
+        } catch(e) {}
+        if (!_origWrite) return undefined;
+        try { return await _origWrite(items); }
+        catch (_) { return undefined; }
+      };
+    })();
   `;
 	await cdp(["eval", tab, code]);
 }
@@ -379,6 +416,79 @@ export function parseSourcesFromMarkdown(text) {
 	return results;
 }
 
+/**
+ * Linear-time "is this a non-empty digit string?" check.
+ * Equivalent to /^\d+$/ without the regex — used to keep the
+ * parseSourcesFromMarkdownRefStyle inline scan free of any regex
+ * (SonarCloud hotspot js:S5852).
+ * @param {string} s
+ * @returns {boolean}
+ */
+function isAllDigits(s) {
+	if (!s) return false;
+	for (let k = 0; k < s.length; k++) {
+		const c = s.charCodeAt(k);
+		if (c < 48 || c > 57) return false;
+	}
+	return true;
+}
+
+/**
+ * Parse reference-style markdown links: [text][num] with [num]: url "title" at bottom.
+ * ChatGPT uses this format for its inline citations.
+ * @param {string} text - Markdown text
+ * @returns {Array<{title: string, url: string}>} Extracted sources
+ */
+export function parseSourcesFromMarkdownRefStyle(text) {
+	if (!text) return [];
+	const results = [];
+
+	// Find all reference definitions: [num]: url "title"
+	const refMap = new Map();
+	const refRegex = /^\[(\d+)\]:\s*(https?:\/\/[^\s"]+)(?:\s+"([^"]*)")?/gm;
+	let m;
+	while ((m = refRegex.exec(text)) !== null) {
+		const num = m[1];
+		const url = m[2];
+		const title = m[3] || "";
+		refMap.set(num, { url, title });
+	}
+
+	// Find inline references: [text][num] or [num]. Linear scan via
+	// indexOf — avoids the ReDoS-prone /\[([^\]]*)\]\[(\d+)\]/g pattern
+	// (SonarCloud hotspot js:S5852). The original `[^\]]*` allowed `[`
+	// inside, which caused quadratic backtracking on inputs like
+	// `[a[[[[[[[[[[[1]`.
+	let cursor = 0;
+	while (cursor < text.length) {
+		const open = text.indexOf("[", cursor);
+		if (open === -1) break;
+		const close = text.indexOf("]", open + 1);
+		if (close === -1) break;
+		if (text[close + 1] !== "[") {
+			cursor = open + 1;
+			continue;
+		}
+		const close2 = text.indexOf("]", close + 2);
+		if (close2 === -1) break;
+
+		const inner = text.slice(open + 1, close);
+		const numStr = text.slice(close + 2, close2);
+		if (isAllDigits(numStr)) {
+			const ref = refMap.get(numStr);
+			if (ref && !results.some((r) => r.url === ref.url)) {
+				results.push({
+					title: inner.trim() || ref.title || "",
+					url: ref.url,
+				});
+			}
+		}
+		cursor = close2 + 1;
+	}
+
+	return results;
+}
+
 // ============================================================================
 // Timing constants
 // ============================================================================
@@ -658,6 +768,26 @@ export function outputJson(data) {
 	process.stdout.write(`${JSON.stringify(data, null, 2)}\n`);
 }
 
+/**
+ * Record the current extractor stage for debugging and timeout diagnostics.
+ * Writes `[engine] stage: <name> (+<ms>)` to stderr and updates `env.lastStage`
+ * / `env.stages` so the envelope carries the last known phase on any outcome
+ * (success, error, timeout, kill).
+ *
+ * @param {object} env - The mutable env object the extractor is filling in.
+ * @param {string} stage - Short, snake_case stage name (e.g. "nav", "type", "stream").
+ * @param {number} [startTime] - Optional extractor start time for elapsed-ms logging.
+ */
+export function logStage(env, stage, startTime = null) {
+	if (!env || typeof env !== "object") return;
+	const elapsed = startTime ? ` (+${Date.now() - startTime}ms)` : "";
+	env.lastStage = stage;
+	if (!Array.isArray(env.stages)) env.stages = [];
+	env.stages.push({ stage, at: Date.now() });
+	const engine = env.engine || "extractor";
+	console.error(`[${engine}] stage: ${stage}${elapsed}`);
+}
+
 /**
  * Build a lightweight result envelope from data already collected during extraction.
  * Zero additional CDP calls — everything here is already known.
@@ -673,6 +803,8 @@ export function buildEnvelope({
 	verificationResult = null,
 	inputReady = null,
 	durationMs = null,
+	lastStage = null,
+	stages = null,
 } = {}) {
 	return {
 		engine,
@@ -683,6 +815,8 @@ export function buildEnvelope({
 		verificationResult,
 		inputReady,
 		durationMs,
+		lastStage,
+		stages,
 	};
 }