@duckmind/dm-windows-x64 0.62.3 → 0.63.0

package/extensions/greedysearch-dm/extractors/consensus.mjs

@@ -1,527 +0,0 @@
-import {
-  buildEnvelope,
-  cdp,
-  cdpWithInput,
-  formatAnswer,
-  getOrOpenTab,
-  handleError,
-  jitter,
-  logStage,
-  outputJson,
-  parseArgs,
-  prepareArgs,
-  TIMING,
-  validateQuery,
-  verifyInputText,
-  waitForSelector
-} from "./common.mjs";
-import { ensureChrome } from "../src/search/chrome.mjs";
-import { dismissConsent } from "./consent.mjs";
-const SELECTORS = {
-  input: 'textarea[name="newMessage"]',
-  submitButton: 'button[aria-label="Submit search"]',
-  answerContainer: 'div[class*="prose"]',
-  paperCard: 'a[data-testid="search-result"]',
-  exportButton: 'button[aria-label="Export"]'
-};
-async function detectSignUpWall(tab) {
-  const code = `(() => {
-		const url = document.location.href || '';
-		return url.indexOf('/sign-up/') !== -1 || url.indexOf('redirect_url=') !== -1;
-	})()`;
-  const result = await cdp(["eval", tab, code]).catch(() => "false");
-  return result === "true";
-}
-async function detectStaleClerkSession(tab) {
-  const code = `(() => {
-		const url = document.location.href || '';
-		const title = document.title || '';
-		const text = document.body?.innerText || '';
-		const stale =
-			url.includes('clerk.consensus.app') ||
-			title.includes('clerk.consensus.app') ||
-			text.includes('session-token-expired') ||
-			text.includes('refresh_request_origin_azp_mismatch') ||
-			(text.includes('HTTP ERROR 405') && text.includes('This page isn'));
-		return JSON.stringify({ stale, url, title, text: text.slice(0, 500) });
-	})()`;
-  try {
-    return JSON.parse(await cdp(["eval", tab, code], 5000));
-  } catch {
-    return { stale: false, url: "", title: "", text: "" };
-  }
-}
-async function clearConsensusAuthStorage(tab) {
-  for (const origin of [
-    "https://consensus.app",
-    "https://clerk.consensus.app"
-  ]) {
-    await cdp([
-      "evalraw",
-      tab,
-      "Storage.clearDataForOrigin",
-      JSON.stringify({ origin, storageTypes: "all" })
-    ]).catch((e) => {
-      console.error(`[consensus] Warning: failed to clear stale auth storage for ${origin}: ${e.message}`);
-    });
-  }
-}
-async function recoverStaleClerkSession(tab, env, startTime) {
-  const before = await detectStaleClerkSession(tab);
-  if (!before.stale)
-    return false;
-  logStage(env, "auth-storage-reset", startTime);
-  console.error(`[consensus] Detected stale Clerk/Consensus auth state (${before.title || before.url}) — clearing per-origin storage and retrying navigation`);
-  env.fallbackUsed = "clear-stale-consensus-auth";
-  await clearConsensusAuthStorage(tab);
-  await cdp(["nav", tab, "https://consensus.app/"], 20000);
-  await new Promise((r) => setTimeout(r, 900));
-  const after = await detectStaleClerkSession(tab);
-  if (after.stale) {
-    env.blockedBy = "signin";
-    env.verificationResult = "needs-human";
-    throw new Error("Consensus auth session is stale — visible Chrome is open. Please sign in again, then rerun the search.");
-  }
-  return true;
-}
-async function typeIntoConsensus(tab, text) {
-  await cdp(["click", tab, SELECTORS.input]);
-  await new Promise((r) => setTimeout(r, jitter(200)));
-  const type = async () => {
-    await cdpWithInput(["type", tab, "--stdin"], text);
-    await new Promise((r) => setTimeout(r, jitter(300)));
-  };
-  const readInput = async () => {
-    const raw = await cdp([
-      "eval",
-      tab,
-      `JSON.stringify(document.querySelector('${SELECTORS.input}')?.value || '')`
-    ]);
-    try {
-      return JSON.parse(raw);
-    } catch {
-      return raw;
-    }
-  };
-  const verify = async () => verifyInputText(await readInput(), text);
-  const clear = async () => {
-    await cdp([
-      "eval",
-      tab,
-      `(() => {
-				const el = document.querySelector('${SELECTORS.input}');
-				if (!el) return false;
-				el.focus();
-				el.select();
-				document.execCommand('delete');
-				return true;
-			})()`
-    ]);
-  };
-  await type();
-  if (await verify())
-    return;
-  await clear();
-  await type();
-  if (!await verify()) {
-    throw new Error("Consensus input did not accept text — input verification failed");
-  }
-}
-async function installPapersDetailsInterceptor(tab) {
-  const code = `(() => {
-		if (window.__pdiInstalled) return 'already';
-		window.__pdiInstalled = true;
-		const _origOpen = XMLHttpRequest.prototype.open;
-		const _origSend = XMLHttpRequest.prototype.send;
-		XMLHttpRequest.prototype.open = function(method, url) {
-			this.__url = String(url);
-			return _origOpen.apply(this, arguments);
-		};
-		XMLHttpRequest.prototype.send = function(body) {
-			if (this.__url && this.__url.indexOf('/api/papers/details') !== -1) {
-				this.addEventListener('load', function() {
-					if (this.status === 200) {
-						try {
-							const parsed = JSON.parse(this.responseText);
-							// Keep only {largest, latest} instead of every
-							// partial Load-More response, to bound page
-							// memory. "latest" lets the caller detect the
-							// .CSV response fired after clickExportCsv;
-							// "largest" is the timeout fallback.
-							const entry = {
-								at: Date.now(),
-								count: Object.keys(parsed?.paperDetailsListByPaperId || {}).length,
-								data: parsed,
-							};
-							const store = window.__papersDetailsResps ||
-								(window.__papersDetailsResps = { largest: null, latest: null });
-							store.latest = entry;
-							if (!store.largest || entry.count > store.largest.count) {
-								store.largest = entry;
-							}
-						} catch (e) {
-							window.__papersDetailsErrors = window.__papersDetailsErrors || [];
-							window.__papersDetailsErrors.push(String(e.message || e));
-						}
-					}
-				});
-			}
-			return _origSend.apply(this, arguments);
-		};
-		return 'installed';
-	})()`;
-  const r = await cdp(["eval", tab, code]);
-  return r === "installed" || r === "already";
-}
-async function fetchPapersDetailsResponse(tab, timeoutMs = 1e4, afterTs = 0) {
-  const code = `new Promise((resolve) => {
-		const _deadline = Date.now() + ${timeoutMs};
-		const _after = ${afterTs};
-		function _check() {
-			const store = window.__papersDetailsResps;
-			const latest = store && store.latest;
-			if (latest && latest.at > _after) {
-				resolve(JSON.stringify({ ok: true, count: latest.count, data: latest.data }));
-				return;
-			}
-			if (Date.now() < _deadline) {
-				setTimeout(_check, 200);
-			} else {
-				const largest = store && store.largest;
-				if (largest) {
-					resolve(JSON.stringify({ ok: true, count: largest.count, data: largest.data, stale: true }));
-				} else {
-					resolve(JSON.stringify({ ok: false, reason: 'timeout' }));
-				}
-			}
-		}
-		_check();
-	})`;
-  const result = await cdp(["eval", tab, code], timeoutMs + 5000);
-  try {
-    return JSON.parse(result);
-  } catch {
-    return { ok: false, reason: "invalid-json" };
-  }
-}
-async function expandReferences(tab, maxClicks = 8) {
-  const ready = await waitForSelector(tab, SELECTORS.paperCard, 20000, 500).catch(() => false);
-  if (!ready) {
-    console.error("[consensus] Warning: no paper cards appeared within 20s — Load More will be skipped");
-    return 0;
-  }
-  const code = String.raw`
-	new Promise((resolve) => {
-		const _paperSel = '${SELECTORS.paperCard}';
-		const _maxClicks = ${maxClicks};
-		function _findLoadMore() {
-			const btns = Array.from(document.querySelectorAll('button'));
-			return btns.find(b => {
-				const t = (b.innerText || '').trim();
-				return /load more/i.test(t) || /more results/i.test(t) || /show more/i.test(t);
-			}) || null;
-		}
-		function _cardCount() { return document.querySelectorAll(_paperSel).length; }
-		let _clicks = 0;
-		function _waitButton(deadline, cb) {
-			const btn = _findLoadMore();
-			if (btn) { cb(btn); return; }
-			if (Date.now() < deadline) { setTimeout(() => _waitButton(deadline, cb), 200); }
-			else { cb(null); }
-		}
-		function _loop() {
-			if (_clicks >= _maxClicks) { resolve(JSON.stringify({ clicks: _clicks, cards: _cardCount() })); return; }
-			const mountDeadline = Date.now() + (_clicks === 0 ? 3000 : 500);
-			_waitButton(mountDeadline, (btn) => {
-				if (!btn || btn.disabled) { resolve(JSON.stringify({ clicks: _clicks, cards: _cardCount() })); return; }
-				const before = _cardCount();
-				btn.click();
-				_clicks++;
-				const clickDeadline = Date.now() + 4000;
-				(function _pollCards() {
-					if (_cardCount() > before || Date.now() >= clickDeadline) { _loop(); return; }
-					setTimeout(_pollCards, 200);
-				})();
-			});
-		}
-		_loop();
-	})
-	`;
-  const result = await cdp(["eval", tab, code], maxClicks * 5000 + 5000).catch(() => null);
-  if (!result)
-    return 0;
-  try {
-    const { clicks, cards } = JSON.parse(result);
-    console.error(`[consensus] expandReferences: ${clicks} click(s), ${cards} card(s)`);
-    return clicks;
-  } catch {
-    return 0;
-  }
-}
-async function waitForCardsStable(tab, maxMs = 1200) {
-  const code = String.raw`
-	new Promise((resolve) => {
-		const _sel = '${SELECTORS.paperCard}';
-		const _deadline = Date.now() + ${maxMs};
-		let _last = document.querySelectorAll(_sel).length;
-		function _poll() {
-			const cur = document.querySelectorAll(_sel).length;
-			if (cur === _last) { resolve(cur); return; }
-			_last = cur;
-			if (Date.now() < _deadline) { setTimeout(_poll, 300); }
-			else { resolve(_last); }
-		}
-		setTimeout(_poll, 300);
-	})
-	`;
-  await cdp(["eval", tab, code], maxMs + 3000).catch(() => null);
-}
-async function clickExportCsv(tab) {
-  await cdp([
-    "eval",
-    tab,
-    `(() => {
-			const btn = document.querySelector('${SELECTORS.exportButton}');
-			if (!btn) return 'no-export';
-			btn.scrollIntoView({ block: 'center' });
-			btn.click();
-			return 'opened';
-		})()`
-  ]);
-  await new Promise((r) => setTimeout(r, 600));
-  const clicked = await cdp([
-    "eval",
-    tab,
-    `(() => {
-			const btn = Array.from(document.querySelectorAll('button'))
-				.find(b => /\\.CSV/.test((b.innerText || '').trim()));
-			if (!btn) return 'no-csv';
-			btn.click();
-			return 'clicked';
-		})()`
-  ]);
-  return clicked;
-}
-function buildSourcesFromApi(respData) {
-  const map = respData?.paperDetailsListByPaperId || {};
-  const ids = Object.keys(map);
-  const sources = [];
-  for (let i = 0;i < ids.length; i++) {
-    const p = map[ids[i]] || {};
-    const urlSlug = p.url_slug || "";
-    const paperId = p.paper_id || p.hash_paper_id || ids[i];
-    let url;
-    if (urlSlug) {
-      url = `https://consensus.app/papers/${urlSlug}/${paperId}/`;
-    } else if (p.provider_url) {
-      url = p.provider_url;
-    } else if (p.doi) {
-      url = `https://doi.org/${p.doi}`;
-    } else {
-      url = `https://consensus.app/paper/${paperId}`;
-    }
-    const tags = [];
-    const badges = p.badges || {};
-    if (badges.study_type === "rct")
-      tags.push("RCT");
-    else if (badges.study_type === "meta_analysis")
-      tags.push("META-ANALYSIS");
-    else if (badges.study_type === "systematic_review")
-      tags.push("SYSTEMATIC REVIEW");
-    else if (badges.study_type)
-      tags.push(badges.study_type.toUpperCase());
-    if (badges.rigorous_journal)
-      tags.push("RIGOROUS JOURNAL");
-    if (badges.very_rigorous_journal)
-      tags.push("VERY RIGOROUS JOURNAL");
-    if (badges.highly_cited_paper)
-      tags.push("HIGHLY CITED");
-    if (badges.large_human_trial)
-      tags.push("LARGE HUMAN TRIAL");
-    if (p.is_retracted)
-      tags.push("RETRACTED");
-    if (p.open_access_pdf_url)
-      tags.push("OPEN ACCESS");
-    sources.push({
-      title: p.title || "",
-      url,
-      rank: i + 1,
-      authors: Array.isArray(p.authors) ? p.authors : [],
-      year: p.year || null,
-      journal: p.journal || p.publisher_name || null,
-      doi: p.doi || null,
-      citation_count: p.citation_count || 0,
-      snippet: p.abstract_takeaway || "",
-      tags
-    });
-  }
-  return sources;
-}
-async function extractSourcesFromDom(tab) {
-  const code = `(() => {
-		const cards = Array.from(document.querySelectorAll('${SELECTORS.paperCard}'));
-		const sources = [];
-		const seen = new Set();
-		for (const card of cards) {
-			const url = card.href || '';
-			if (!url || seen.has(url)) continue;
-			seen.add(url);
-			const rankText = card.querySelector('span[data-testid="tag"]')?.innerText?.trim() || '';
-			const rank = parseInt(rankText, 10) || null;
-			const title = (card.querySelector('h2')?.innerText || '').trim();
-			// Key takeaway: the span after the "KEY TAKEAWAY" label
-			const takeawaySpan = card.querySelector('span.sm-normal');
-			const snippet = takeawaySpan
-				? takeawaySpan.innerText.replace(/^KEY TAKEAWAY\\s*[·\\-]\\s*/i, '').trim()
-				: '';
-			// Tags: collect chip text from meta-analysis-tag, journal tags, etc.
-			const tags = Array.from(card.querySelectorAll('span[data-testid$="-tag"]'))
-				.map(t => t.innerText?.trim())
-				.filter(Boolean);
-			sources.push({ title, url, rank, snippet, tags });
-		}
-		return JSON.stringify(sources);
-	})()`;
-  const result = await cdp(["eval", tab, code], 1e4);
-  try {
-    return JSON.parse(result);
-  } catch {
-    return [];
-  }
-}
-async function extractAnswer(tab) {
-  const code = `(() => {
-		const prose = document.querySelector('${SELECTORS.answerContainer}');
-		return prose?.innerText?.trim() || '';
-	})()`;
-  return await cdp(["eval", tab, code], 1e4) || "";
-}
-const USAGE = `Usage: node extractors/consensus.mjs "<query>" [--tab <prefix>]
-`;
-async function main() {
-  const args = await prepareArgs(process.argv.slice(2));
-  validateQuery(args, USAGE);
-  const { query, tabPrefix, short } = parseArgs(args);
-  const startTime = Date.now();
-  const mode = process.env.GREEDY_SEARCH_VISIBLE === "1" ? "visible" : "headless";
-  const env = {
-    engine: "consensus",
-    mode,
-    clipboardEmpty: null,
-    fallbackUsed: null,
-    blockedBy: null,
-    verificationResult: null,
-    inputReady: null
-  };
-  try {
-    if (process.env.GREEDY_SEARCH_VISIBLE !== "1" && process.env.GREEDY_SEARCH_ALWAYS_VISIBLE !== "1") {
-      process.env.GREEDY_SEARCH_HEADLESS = "1";
-    }
-    await ensureChrome();
-    if (!tabPrefix)
-      await cdp(["list"]);
-    const tab = await getOrOpenTab(tabPrefix);
-    const currentUrl = await cdp(["eval", tab, "document.location.href"]).catch(() => "");
-    let onConsensus = false;
-    try {
-      const host = new URL(currentUrl).hostname.toLowerCase();
-      onConsensus = host === "consensus.app" || host.endsWith(".consensus.app");
-    } catch {}
-    if (!onConsensus) {
-      logStage(env, "nav", startTime);
-      await cdp(["nav", tab, "https://consensus.app/"], 20000);
-      await new Promise((r) => setTimeout(r, 600));
-    }
-    await recoverStaleClerkSession(tab, env, startTime);
-    await dismissConsent(tab, cdp);
-    logStage(env, "input-wait", startTime);
-    const inputReady = await waitForSelector(tab, SELECTORS.input, 15000, 400);
-    env.inputReady = inputReady;
-    if (!inputReady) {
-      const recovered = await recoverStaleClerkSession(tab, env, startTime);
-      if (recovered) {
-        const retryInputReady = await waitForSelector(tab, SELECTORS.input, 15000, 400);
-        env.inputReady = retryInputReady;
-        if (retryInputReady) {
-          await dismissConsent(tab, cdp);
-        } else {
-          throw new Error("Consensus input not found after stale auth recovery — page may not have loaded or is in unexpected state");
-        }
-      } else {
-        throw new Error("Consensus input not found — page may not have loaded or is in unexpected state");
-      }
-    }
-    await new Promise((r) => setTimeout(r, jitter(TIMING.postClick)));
-    logStage(env, "type-and-submit", startTime);
-    await typeIntoConsensus(tab, query);
-    await new Promise((r) => setTimeout(r, jitter(TIMING.postType)));
-    await cdp([
-      "eval",
-      tab,
-      `document.querySelector('${SELECTORS.submitButton}')?.click()`
-    ]);
-    await new Promise((r) => setTimeout(r, 2000));
-    if (await detectSignUpWall(tab)) {
-      env.blockedBy = "signin";
-      throw new Error("Consensus requires sign-in — please sign in or create a free account in the visible browser window. Once signed in, cookies persist for future runs.");
-    }
-    logStage(env, "answer-wait", startTime);
-    await waitForSelector(tab, SELECTORS.answerContainer, 30000, 500);
-    await installPapersDetailsInterceptor(tab);
-    logStage(env, "expand-refs", startTime);
-    const clicks = await expandReferences(tab, 8);
-    if (clicks === 0) {
-      console.error("[consensus] Note: 'Load more results' button not present (initial page has all references)");
-    } else {
-      console.error(`[consensus] Clicked 'Load more results' ${clicks} time(s) to expand references`);
-    }
-    await waitForCardsStable(tab, 1200);
-    logStage(env, "csv-click", startTime);
-    const preClickTs = Number(await cdp(["eval", tab, "Date.now()"]).catch(() => "0") || 0);
-    const csvResult = await clickExportCsv(tab);
-    if (csvResult !== "clicked") {
-      console.error(`[consensus] Export → .CSV click did not register (${csvResult}) — falling back to DOM`);
-    }
-    logStage(env, "wait-csv-resp", startTime);
-    const csvResp = await fetchPapersDetailsResponse(tab, 12000, preClickTs);
-    let sources = [];
-    let sourcePath = "dom";
-    if (csvResp.ok) {
-      sources = buildSourcesFromApi(csvResp.data);
-      sourcePath = "api-intercept";
-      console.error(`[consensus] Captured /api/papers/details/ response with ${csvResp.count} paper(s)`);
-    } else {
-      console.error(`[consensus] /api/papers/details/ response not captured (${csvResp.reason}) — falling back to DOM cards`);
-    }
-    logStage(env, "extract", startTime);
-    const answer = await extractAnswer(tab);
-    if (!answer) {
-      throw new Error("No answer extracted — Consensus may not have responded");
-    }
-    if (sources.length === 0) {
-      sources = await extractSourcesFromDom(tab);
-    } else {
-      const domSources = await extractSourcesFromDom(tab);
-      const apiUrls = new Set(sources.map((s) => s.url));
-      for (const ds of domSources) {
-        if (!apiUrls.has(ds.url))
-          sources.push(ds);
-      }
-    }
-    const finalUrl = await cdp(["eval", tab, "document.location.href"]).catch(() => "");
-    env.durationMs = Date.now() - startTime;
-    env.sourcePath = sourcePath;
-    logStage(env, "done", startTime);
-    outputJson({
-      query,
-      url: finalUrl,
-      answer: formatAnswer(answer, short),
-      sources,
-      _envelope: buildEnvelope(env)
-    });
-  } catch (e) {
-    env.durationMs = Date.now() - startTime;
-    handleError(e, buildEnvelope(env));
-  }
-}
-main();