@duckmind/dm-windows-x64 0.60.6 → 0.60.9

package/extensions/greedysearch-dm/src/fetcher.mjs

@@ -1,3 +1,577 @@
-import{createRequire as y}from"node:module";var C=y(import.meta.url);var L=null;async function A(){if(L)return L;let[{Readability:K},{JSDOM:Y},{default:j}]=await Promise.all([import("@mozilla/readability"),import("jsdom"),import("turndown")]),Z=new j({headingStyle:"atx",bulletListMarker:"-",codeBlockStyle:"fenced"});return Z.addRule("removeDataUrls",{filter:($)=>$.tagName==="IMG"&&$.getAttribute("src")?.startsWith("data:"),replacement:()=>""}),L={Readability:K,JSDOM:Y,turndown:Z},L}var w="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",g={"user-agent":w,accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","accept-encoding":"gzip, deflate, br","cache-control":"no-cache",pragma:"no-cache","sec-ch-ua":'"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"Windows"',"sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"none","sec-fetch-user":"?1","upgrade-insecure-requests":"1"},k=[/^localhost$/i,/^127\.\d+\.\d+\.\d+$/,/^0\.0\.0\.0$/,/^\[::1\]$/,/^10\./,/^172\.(1[6-9]|2\d|3[01])\./,/^192\.168\./,/^169\.254\./,/^fc00:/i,/^fe80:/i,/\.local$/i,/\.internal$/i,/\.localhost$/i],P=/^::ffff:(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/i,R=/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i,_=/^(?:0x[0-9a-f]+|0[0-7]*|[1-9][0-9]*)$/i;function M(K){if(!K||K.includes(":"))return null;let Y=K.split(".");if(Y.length<1||Y.length>4)return null;if(!Y.every(($)=>_.test($)))return null;let j=Y.map(($)=>{if(/^0x/i.test($))return parseInt($,16);if(/^0[0-7]+$/.test($))return parseInt($,8);return parseInt($,10)});if(j.some(($)=>!Number.isFinite($)||$<0))return null;let Z;if(j.length===1){if(j[0]>4294967295)return null;Z=[j[0]>>>24&255,j[0]>>>16&255,j[0]>>>8&255,j[0]&255]}else if(j.length===2){if(j[0]>255||j[1]>16777215)return null;Z=[j[0],j[1]>>>16&255,j[1]>>>8&255,j[1]&255]}else if(j.length===3){if(j[0]>255||j[1]>255||j[2]>65535)return null;Z=[j[0],j[1],j[2]>>>8&255,j[2]&255]}else{if(j.some(($)=>$>255))return null;Z=j}return Z.join(".")}function E(K){let Y=K.match(P);if(Y)return`${Y[1]}.${Y[2]}.${Y[3]}.${Y[4]}`;let j=K.match(R);if(j){let Z=parseInt(j[1],16),$=parseInt(j[2],16);if(Z>65535||$>65535)return null;return[Z>>>8&255,Z&255,$>>>8&255,$&255].join(".")}return null}function D(K){return k.some((Y)=>Y.test(K))}function o(K={}){return{...g,...K}}function b(K){try{if(typeof K!=="string"||!K.trim())return{blocked:!0,reason:"URL must be a non-empty string"};let Y=new URL(K);if(Y.protocol!=="http:"&&Y.protocol!=="https:")return{blocked:!0,reason:`Protocol not allowed: ${Y.protocol}`};let j=Y.hostname.toLowerCase();for(let $ of k)if($.test(j))return{blocked:!0,reason:`Private/internal address: ${j}`};if(j.startsWith("[")&&j.endsWith("]")){let $=j.slice(1,-1),W=E($);if(W&&D(W))return{blocked:!0,reason:`Private/internal address: ${j} (maps to ${W})`}}let Z=M(j);if(Z&&D(Z))return{blocked:!0,reason:`Private/internal address: ${j} (normalizes to ${Z})`};return{blocked:!1}}catch(Y){return{blocked:!0,reason:`Invalid URL: ${Y.message}`}}}function x(K){try{let Y=new URL(K);if(!(Y.hostname==="github.com"||Y.hostname.endsWith(".github.com")))return K;let j=Y.pathname.split("/").filter(Boolean);if(j.length<5)return K;let[Z,$,W,X,...z]=j;if(W!=="blob")return K;let J=z.join("/");return`https://raw.githubusercontent.com/${Z}/${$}/${X}/${J}`}catch{return K}}async function n(K,Y={}){let j=b(K);if(j.blocked)return{ok:!1,url:K,finalUrl:K,status:403,error:`Blocked: ${j.reason}`,needsBrowser:!1};let Z=K;if(K=x(K),K!==Z)console.error(`[fetcher] Rewrote GitHub URL: ${Z.slice(0,60)}... → raw.githubusercontent.com`);let{timeoutMs:$=15000,userAgent:W,signal:X}=Y,z=new AbortController,J=setTimeout(()=>z.abort(),$);if(X)X.addEventListener("abort",()=>z.abort(),{once:!0});try{let Q=await fetch(K,{method:"GET",headers:{...g,"user-agent":W||w},redirect:"follow",signal:z.signal});clearTimeout(J);let B=Q.headers.get("content-type")||"",V=Q.url,F=Q.headers.get("last-modified")||"",q=b(V);if(q.blocked)return{ok:!1,url:K,finalUrl:V,status:Q.status,error:`Blocked: ${q.reason}`,needsBrowser:!1};let H=!1;try{H=new URL(V).hostname.toLowerCase()==="raw.githubusercontent.com"}catch{}if(B.includes("text/plain")&&H){let O=await Q.text();return{ok:!0,url:Z,finalUrl:V,status:Q.status,title:V.split("/").pop()||"GitHub File",byline:"",siteName:"GitHub",lang:"",publishedTime:F,lastModified:F,markdown:O,contentLength:O.length,excerpt:O.slice(0,300).replaceAll(/\n/g," "),needsBrowser:!1}}if(!B.includes("text/html")&&!B.includes("application/xhtml"))return{ok:!1,url:K,finalUrl:V,status:Q.status,error:`Unsupported content type: ${B}`,needsBrowser:!1};let I=await Q.text(),N=T(Q.status,I,V,K);if(N.blocked)return{ok:!1,url:K,finalUrl:V,status:Q.status,error:`Blocked: ${N.reason}`,needsBrowser:!0};let G=await c(I,V),S=d(G);if(!S.ok)return{ok:!1,url:K,finalUrl:V,status:Q.status,error:`Low quality content: ${S.reason}`,needsBrowser:!0};return{ok:!0,url:K,finalUrl:V,status:Q.status,title:G.title,byline:G.byline,siteName:G.siteName,lang:G.lang,publishedTime:G.publishedTime||F,lastModified:F,markdown:G.markdown,excerpt:G.excerpt,contentLength:G.markdown.length,needsBrowser:!1}}catch(Q){clearTimeout(J);let B=m(Q);return{ok:!1,url:K,finalUrl:K,status:0,error:Q.message,needsBrowser:B}}}function T(K,Y,j,Z){let $=Y.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.toLowerCase()||"",W=Y.slice(0,30000).toLowerCase(),X=`${$} ${W}`;if(K===403||K===429||K===503)return{blocked:!0,reason:`HTTP ${K}`};let z=[{pattern:/class=["'][^"']*captcha["']|<div[^>]*id=["']captcha/i,reason:"captcha"},{pattern:/g-recaptcha|data-sitekey|i['"]m not a robot/i,reason:"captcha"},{pattern:/checking your browser.{0,100}please wait|cf-browser-verification/i,reason:"cloudflare challenge"},{pattern:/just a moment.{0,50}security check|ddos protection by cloudflare/i,reason:"cloudflare challenge"},{pattern:/unusual traffic.{0,50}from your computer network/i,reason:"unusual traffic"},{pattern:/bot detected|automated.{0,20}request/i,reason:"bot detection"},{pattern:/enable\s+javascript\s+to\s+view|javascript\s+is\s+required.{0,50}enabled/i,reason:"requires javascript"},{pattern:/access denied|accessdenied/i,reason:"access denied"},{pattern:/protected by anubis|anubis uses a proof-of-work/i,reason:"anubis challenge"}];for(let Q of z)if(Q.pattern.test(X))return{blocked:!0,reason:Q.reason};let J=p(Z,j,Y);if(J)return{blocked:!0,reason:J};return{blocked:!1}}var f=["accounts.google.com","login.microsoftonline.com","login.live.com","auth0.com","okta.com","auth.mozilla.auth0.com","id.atlassian.com"],U=["login.","signin.","auth.","sso.","accounts.","idp."],h=["sign in to continue","log in to continue","authentication required","create an account to continue","subscribe to continue reading","members only"];function p(K,Y,j){try{let Z=new URL(K),$=new URL(Y);if(Z.hostname.toLowerCase()===$.hostname.toLowerCase())return;let W=$.hostname.toLowerCase();if(f.some((z)=>W===z||W.endsWith(`.${z}`)))return`redirected to login (${$.hostname})`;if(U.some((z)=>W.startsWith(z)))return`redirected to login (${$.hostname})`;let X=j.slice(0,20000).toLowerCase();if(h.some((z)=>X.includes(z)))return`redirected to login page (${$.hostname})`}catch{}return}function m(K){let Y=K.message.toLowerCase();return Y.includes("fetch failed")||Y.includes("unable to verify")||Y.includes("certificate")||Y.includes("timeout")}function v(K){let Y=['meta[property="article:published_time"]','meta[name="article:published_time"]','meta[property="og:published_time"]','meta[name="publication_date"]','meta[name="date"]','meta[itemprop="datePublished"]','time[itemprop="datePublished"]','meta[name="DC.date"]'];for(let j of Y){let Z=K.querySelector(j),$=Z?.getAttribute("content")||Z?.getAttribute("datetime")||"";if($)return $}return""}async function c(K,Y){let{Readability:j,JSDOM:Z,turndown:$}=await A(),W=new Z(K,{url:Y});try{let X=W.window.document,J=new j(X).parse();if(J&&J.content){let V=$.turndown(J.content).replaceAll(/\n{3,}/g,`
+let _deps = null;
+async function loadDeps() {
+  if (_deps)
+    return _deps;
+  const [{ Readability }, { JSDOM }, { default: TurndownService }] = await Promise.all([
+    import("@mozilla/readability"),
+    import("jsdom"),
+    import("turndown")
+  ]);
+  const turndown = new TurndownService({
+    headingStyle: "atx",
+    bulletListMarker: "-",
+    codeBlockStyle: "fenced"
+  });
+  turndown.addRule("removeDataUrls", {
+    filter: (node) => node.tagName === "IMG" && node.getAttribute("src")?.startsWith("data:"),
+    replacement: () => ""
+  });
+  _deps = { Readability, JSDOM, turndown };
+  return _deps;
+}
+const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
+const DEFAULT_HEADERS = {
+  "user-agent": DEFAULT_USER_AGENT,
+  accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+  "accept-language": "en-US,en;q=0.9",
+  "accept-encoding": "gzip, deflate, br",
+  "cache-control": "no-cache",
+  pragma: "no-cache",
+  "sec-ch-ua": '"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',
+  "sec-ch-ua-mobile": "?0",
+  "sec-ch-ua-platform": '"Windows"',
+  "sec-fetch-dest": "document",
+  "sec-fetch-mode": "navigate",
+  "sec-fetch-site": "none",
+  "sec-fetch-user": "?1",
+  "upgrade-insecure-requests": "1"
+};
+const PRIVATE_URL_PATTERNS = [
+  /^localhost$/i,
+  /^127\.\d+\.\d+\.\d+$/,
+  /^0\.0\.0\.0$/,
+  /^\[::1\]$/,
+  /^10\./,
+  /^172\.(1[6-9]|2\d|3[01])\./,
+  /^192\.168\./,
+  /^169\.254\./,
+  /^fc00:/i,
+  /^fe80:/i,
+  /\.local$/i,
+  /\.internal$/i,
+  /\.localhost$/i
+];
+const IPV4_MAPPED_DOTTED_RE = /^::ffff:(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/i;
+const IPV4_MAPPED_HEX_RE = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i;
+const INET_ATON_SEGMENT_RE = /^(?:0x[0-9a-f]+|0[0-7]*|[1-9][0-9]*)$/i;
+function normalizeNumericIPv4(hostname) {
+  if (!hostname || hostname.includes(":"))
+    return null;
+  const parts = hostname.split(".");
+  if (parts.length < 1 || parts.length > 4)
+    return null;
+  if (!parts.every((p) => INET_ATON_SEGMENT_RE.test(p)))
+    return null;
+  const nums = parts.map((p) => {
+    if (/^0x/i.test(p))
+      return parseInt(p, 16);
+    if (/^0[0-7]+$/.test(p))
+      return parseInt(p, 8);
+    return parseInt(p, 10);
+  });
+  if (nums.some((n) => !Number.isFinite(n) || n < 0))
+    return null;
+  let bytes;
+  if (nums.length === 1) {
+    if (nums[0] > 4294967295)
+      return null;
+    bytes = [
+      nums[0] >>> 24 & 255,
+      nums[0] >>> 16 & 255,
+      nums[0] >>> 8 & 255,
+      nums[0] & 255
+    ];
+  } else if (nums.length === 2) {
+    if (nums[0] > 255 || nums[1] > 16777215)
+      return null;
+    bytes = [nums[0], nums[1] >>> 16 & 255, nums[1] >>> 8 & 255, nums[1] & 255];
+  } else if (nums.length === 3) {
+    if (nums[0] > 255 || nums[1] > 255 || nums[2] > 65535)
+      return null;
+    bytes = [nums[0], nums[1], nums[2] >>> 8 & 255, nums[2] & 255];
+  } else {
+    if (nums.some((n) => n > 255))
+      return null;
+    bytes = nums;
+  }
+  return bytes.join(".");
+}
+function extractMappedIPv4(core) {
+  const dottedMatch = core.match(IPV4_MAPPED_DOTTED_RE);
+  if (dottedMatch) {
+    return `${dottedMatch[1]}.${dottedMatch[2]}.${dottedMatch[3]}.${dottedMatch[4]}`;
+  }
+  const hexMatch = core.match(IPV4_MAPPED_HEX_RE);
+  if (hexMatch) {
+    const hi = parseInt(hexMatch[1], 16);
+    const lo = parseInt(hexMatch[2], 16);
+    if (hi > 65535 || lo > 65535)
+      return null;
+    return [
+      hi >>> 8 & 255,
+      hi & 255,
+      lo >>> 8 & 255,
+      lo & 255
+    ].join(".");
+  }
+  return null;
+}
+function isPrivateIPv4(ip) {
+  return PRIVATE_URL_PATTERNS.some((pattern) => pattern.test(ip));
+}
+export function defaultFetchHeaders(overrides = {}) {
+  return { ...DEFAULT_HEADERS, ...overrides };
+}
+export function isPrivateUrl(url) {
+  try {
+    if (typeof url !== "string" || !url.trim()) {
+      return { blocked: true, reason: "URL must be a non-empty string" };
+    }
+    const parsed = new URL(url);
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+      return {
+        blocked: true,
+        reason: `Protocol not allowed: ${parsed.protocol}`
+      };
+    }
+    const hostname = parsed.hostname.toLowerCase();
+    for (const pattern of PRIVATE_URL_PATTERNS) {
+      if (pattern.test(hostname)) {
+        return {
+          blocked: true,
+          reason: `Private/internal address: ${hostname}`
+        };
+      }
+    }
+    if (hostname.startsWith("[") && hostname.endsWith("]")) {
+      const core = hostname.slice(1, -1);
+      const mapped = extractMappedIPv4(core);
+      if (mapped && isPrivateIPv4(mapped)) {
+        return {
+          blocked: true,
+          reason: `Private/internal address: ${hostname} (maps to ${mapped})`
+        };
+      }
+    }
+    const normalized = normalizeNumericIPv4(hostname);
+    if (normalized && isPrivateIPv4(normalized)) {
+      return {
+        blocked: true,
+        reason: `Private/internal address: ${hostname} (normalizes to ${normalized})`
+      };
+    }
+    return { blocked: false };
+  } catch (error) {
+    return { blocked: true, reason: `Invalid URL: ${error.message}` };
+  }
+}
+export function rewriteGitHubUrl(url) {
+  try {
+    const parsed = new URL(url);
+    if (!(parsed.hostname === "github.com" || parsed.hostname.endsWith(".github.com"))) {
+      return url;
+    }
+    const parts = parsed.pathname.split("/").filter(Boolean);
+    if (parts.length < 5) {
+      return url;
+    }
+    const [owner, repo, type, ref, ...fileParts] = parts;
+    if (type !== "blob") {
+      return url;
+    }
+    const rawPath = fileParts.join("/");
+    const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${rawPath}`;
+    return rawUrl;
+  } catch {
+    return url;
+  }
+}
+export async function fetchSourceHttp(url, options = {}) {
+  const privateCheck = isPrivateUrl(url);
+  if (privateCheck.blocked) {
+    return {
+      ok: false,
+      url,
+      finalUrl: url,
+      status: 403,
+      error: `Blocked: ${privateCheck.reason}`,
+      needsBrowser: false
+    };
+  }
+  const originalUrl = url;
+  url = rewriteGitHubUrl(url);
+  if (url !== originalUrl) {
+    console.error(`[fetcher] Rewrote GitHub URL: ${originalUrl.slice(0, 60)}... → raw.githubusercontent.com`);
+  }
+  const { timeoutMs = 15000, userAgent, signal } = options;
+  const controller = new AbortController;
+  const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+  if (signal) {
+    signal.addEventListener("abort", () => controller.abort(), { once: true });
+  }
+  try {
+    const response = await fetch(url, {
+      method: "GET",
+      headers: {
+        ...DEFAULT_HEADERS,
+        "user-agent": userAgent || DEFAULT_USER_AGENT
+      },
+      redirect: "follow",
+      signal: controller.signal
+    });
+    clearTimeout(timeoutId);
+    const contentType = response.headers.get("content-type") || "";
+    const finalUrl = response.url;
+    const lastModified = response.headers.get("last-modified") || "";
+    const finalPrivateCheck = isPrivateUrl(finalUrl);
+    if (finalPrivateCheck.blocked) {
+      return {
+        ok: false,
+        url,
+        finalUrl,
+        status: response.status,
+        error: `Blocked: ${finalPrivateCheck.reason}`,
+        needsBrowser: false
+      };
+    }
+    let isRawGitHub = false;
+    try {
+      const finalHost = new URL(finalUrl).hostname.toLowerCase();
+      isRawGitHub = finalHost === "raw.githubusercontent.com";
+    } catch {}
+    if (contentType.includes("text/plain") && isRawGitHub) {
+      const text = await response.text();
+      return {
+        ok: true,
+        url: originalUrl,
+        finalUrl,
+        status: response.status,
+        title: finalUrl.split("/").pop() || "GitHub File",
+        byline: "",
+        siteName: "GitHub",
+        lang: "",
+        publishedTime: lastModified,
+        lastModified,
+        markdown: text,
+        contentLength: text.length,
+        excerpt: text.slice(0, 300).replaceAll(/\n/g, " "),
+        needsBrowser: false
+      };
+    }
+    if (!contentType.includes("text/html") && !contentType.includes("application/xhtml")) {
+      return {
+        ok: false,
+        url,
+        finalUrl,
+        status: response.status,
+        error: `Unsupported content type: ${contentType}`,
+        needsBrowser: false
+      };
+    }
+    const html = await response.text();
+    const quickCheck = detectBotBlock(response.status, html, finalUrl, url);
+    if (quickCheck.blocked) {
+      return {
+        ok: false,
+        url,
+        finalUrl,
+        status: response.status,
+        error: `Blocked: ${quickCheck.reason}`,
+        needsBrowser: true
+      };
+    }
+    const extracted = await extractContent(html, finalUrl);
+    const quality = checkContentQuality(extracted);
+    if (!quality.ok) {
+      return {
+        ok: false,
+        url,
+        finalUrl,
+        status: response.status,
+        error: `Low quality content: ${quality.reason}`,
+        needsBrowser: true
+      };
+    }
+    return {
+      ok: true,
+      url,
+      finalUrl,
+      status: response.status,
+      title: extracted.title,
+      byline: extracted.byline,
+      siteName: extracted.siteName,
+      lang: extracted.lang,
+      publishedTime: extracted.publishedTime || lastModified,
+      lastModified,
+      markdown: extracted.markdown,
+      excerpt: extracted.excerpt,
+      contentLength: extracted.markdown.length,
+      needsBrowser: false
+    };
+  } catch (error) {
+    clearTimeout(timeoutId);
+    const needsBrowser = isNetworkErrorRetryableWithBrowser(error);
+    return {
+      ok: false,
+      url,
+      finalUrl: url,
+      status: 0,
+      error: error.message,
+      needsBrowser
+    };
+  }
+}
+export function detectBotBlock(status, html, finalUrl, originalUrl) {
+  const title = html.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.toLowerCase() || "";
+  const sample = html.slice(0, 30000).toLowerCase();
+  const combined = `${title} ${sample}`;
+  if (status === 403 || status === 429 || status === 503) {
+    return { blocked: true, reason: `HTTP ${status}` };
+  }
+  const blockSignals = [
+    {
+      pattern: /class=["'][^"']*captcha["']|<div[^>]*id=["']captcha/i,
+      reason: "captcha"
+    },
+    {
+      pattern: /g-recaptcha|data-sitekey|i['"]m not a robot/i,
+      reason: "captcha"
+    },
+    {
+      pattern: /checking your browser.{0,100}please wait|cf-browser-verification/i,
+      reason: "cloudflare challenge"
+    },
+    {
+      pattern: /just a moment.{0,50}security check|ddos protection by cloudflare/i,
+      reason: "cloudflare challenge"
+    },
+    {
+      pattern: /unusual traffic.{0,50}from your computer network/i,
+      reason: "unusual traffic"
+    },
+    {
+      pattern: /bot detected|automated.{0,20}request/i,
+      reason: "bot detection"
+    },
+    {
+      pattern: /enable\s+javascript\s+to\s+view|javascript\s+is\s+required.{0,50}enabled/i,
+      reason: "requires javascript"
+    },
+    { pattern: /access denied|accessdenied/i, reason: "access denied" },
+    {
+      pattern: /protected by anubis|anubis uses a proof-of-work/i,
+      reason: "anubis challenge"
+    }
+  ];
+  for (const signal of blockSignals) {
+    if (signal.pattern.test(combined)) {
+      return { blocked: true, reason: signal.reason };
+    }
+  }
+  const loginRedirect = detectLoginRedirect(originalUrl, finalUrl, html);
+  if (loginRedirect) {
+    return { blocked: true, reason: loginRedirect };
+  }
+  return { blocked: false };
+}
+const AUTH_DOMAINS = [
+  "accounts.google.com",
+  "login.microsoftonline.com",
+  "login.live.com",
+  "auth0.com",
+  "okta.com",
+  "auth.mozilla.auth0.com",
+  "id.atlassian.com"
+];
+const AUTH_HOSTNAME_PREFIXES = [
+  "login.",
+  "signin.",
+  "auth.",
+  "sso.",
+  "accounts.",
+  "idp."
+];
+const LOGIN_CONTENT_PATTERNS = [
+  "sign in to continue",
+  "log in to continue",
+  "authentication required",
+  "create an account to continue",
+  "subscribe to continue reading",
+  "members only"
+];
+function detectLoginRedirect(requestedUrl, finalUrl, html) {
+  try {
+    const requested = new URL(requestedUrl);
+    const final = new URL(finalUrl);
+    if (requested.hostname.toLowerCase() === final.hostname.toLowerCase()) {
+      return;
+    }
+    const finalHost = final.hostname.toLowerCase();
+    if (AUTH_DOMAINS.some((d) => finalHost === d || finalHost.endsWith(`.${d}`))) {
+      return `redirected to login (${final.hostname})`;
+    }
+    if (AUTH_HOSTNAME_PREFIXES.some((p) => finalHost.startsWith(p))) {
+      return `redirected to login (${final.hostname})`;
+    }
+    const sample = html.slice(0, 20000).toLowerCase();
+    if (LOGIN_CONTENT_PATTERNS.some((p) => sample.includes(p))) {
+      return `redirected to login page (${final.hostname})`;
+    }
+  } catch {}
+  return;
+}
+function isNetworkErrorRetryableWithBrowser(error) {
+  const message = error.message.toLowerCase();
+  return message.includes("fetch failed") || message.includes("unable to verify") || message.includes("certificate") || message.includes("timeout");
+}
+function extractMetaDate(document) {
+  const selectors = [
+    'meta[property="article:published_time"]',
+    'meta[name="article:published_time"]',
+    'meta[property="og:published_time"]',
+    'meta[name="publication_date"]',
+    'meta[name="date"]',
+    'meta[itemprop="datePublished"]',
+    'time[itemprop="datePublished"]',
+    'meta[name="DC.date"]'
+  ];
+  for (const sel of selectors) {
+    const el = document.querySelector(sel);
+    const val = el?.getAttribute("content") || el?.getAttribute("datetime") || "";
+    if (val)
+      return val;
+  }
+  return "";
+}
+export async function extractContent(html, url) {
+  const { Readability, JSDOM, turndown } = await loadDeps();
+  const dom = new JSDOM(html, { url });
+  try {
+    const document = dom.window.document;
+    const reader = new Readability(document);
+    const article = reader.parse();
+    if (article && article.content) {
+      const markdown = turndown.turndown(article.content);
+      const cleanMarkdown = markdown.replaceAll(/\n{3,}/g, `
 
-`).trim(),F=J.publishedTime||v(X)||"";return{title:J.title||X.title||Y,byline:J.byline||"",siteName:J.siteName||"",lang:J.lang||"",publishedTime:F,markdown:V,excerpt:V.slice(0,300).replaceAll(/\n/g," ")}}let Q=X.body;if(Q){let B=Q.cloneNode(!0);B.querySelectorAll("script, style, nav, footer, header, aside").forEach((q)=>q.remove());let F=(B.textContent||"").replaceAll(/\s+/g," ").trim();return{title:X.title||Y,byline:"",siteName:"",lang:"",publishedTime:v(X),markdown:F,excerpt:F.slice(0,300)}}return{title:Y,byline:"",siteName:"",lang:"",publishedTime:"",markdown:"",excerpt:""}}finally{W.window.close()}}function d(K){let Y=K.markdown.trim().toLowerCase(),j=(K.title||"").toLowerCase();if(K.markdown.trim().length<100)return{ok:!1,reason:"content too short (< 100 chars)"};let Z=Y.toLowerCase(),$=[{check:()=>Z.includes("loading")&&Z.includes("please wait"),desc:"loading page"},{check:()=>Z.includes("please ensure javascript is enabled"),desc:"requires javascript"},{check:()=>Z.includes("enable javascript to view"),desc:"requires javascript"},{check:()=>Z.includes("just a moment"),desc:"cloudflare challenge detected in content"},{check:()=>Z.includes("verify you are human"),desc:"human verification"},{check:()=>Z.includes("captcha required"),desc:"captcha in extracted content"},{check:()=>Z.includes("access denied"),desc:"access denied in content"},{check:()=>/^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(Y),desc:"login form only"}];for(let{check:W,desc:X}of $)if(W())return{ok:!1,reason:X};if(j.includes("just a moment")||j.includes("checking your browser"))return{ok:!1,reason:"cloudflare challenge page detected in title"};return{ok:!0}}function s(K){try{let Y=new URL(K),j=Y.hostname.toLowerCase(),Z=Y.pathname.toLowerCase();if(["react.dev","nextjs.org","vuejs.org","angular.io","svelte.dev","docs.expo.dev","tailwindcss.com","storybook.js.org"].some((W)=>j===W||j.endsWith(`.${W}`)))return!0;if(Z.includes("/playground")||Z.includes("/demo")||Z.includes("/app"))return!0;if(Y.hash&&Y.hash.length>1)return!0;return!1}catch{return!1}}export{s as shouldUseBrowser,x as rewriteGitHubUrl,b as isPrivateUrl,n as fetchSourceHttp,c as extractContent,T as detectBotBlock,o as defaultFetchHeaders,d as checkContentQuality};
+`).trim();
+      const publishedTime = article.publishedTime || extractMetaDate(document) || "";
+      return {
+        title: article.title || document.title || url,
+        byline: article.byline || "",
+        siteName: article.siteName || "",
+        lang: article.lang || "",
+        publishedTime,
+        markdown: cleanMarkdown,
+        excerpt: cleanMarkdown.slice(0, 300).replaceAll(/\n/g, " ")
+      };
+    }
+    const body = document.body;
+    if (body) {
+      const clone = body.cloneNode(true);
+      clone.querySelectorAll("script, style, nav, footer, header, aside").forEach((el) => el.remove());
+      const text = clone.textContent || "";
+      const cleanText = text.replaceAll(/\s+/g, " ").trim();
+      return {
+        title: document.title || url,
+        byline: "",
+        siteName: "",
+        lang: "",
+        publishedTime: extractMetaDate(document),
+        markdown: cleanText,
+        excerpt: cleanText.slice(0, 300)
+      };
+    }
+    return {
+      title: url,
+      byline: "",
+      siteName: "",
+      lang: "",
+      publishedTime: "",
+      markdown: "",
+      excerpt: ""
+    };
+  } finally {
+    dom.window.close();
+  }
+}
+export function checkContentQuality(extracted) {
+  const markdown = extracted.markdown.trim().toLowerCase();
+  const title = (extracted.title || "").toLowerCase();
+  if (extracted.markdown.trim().length < 100) {
+    return { ok: false, reason: "content too short (< 100 chars)" };
+  }
+  const lc = markdown.toLowerCase();
+  const suspiciousChecks = [
+    {
+      check: () => lc.includes("loading") && lc.includes("please wait"),
+      desc: "loading page"
+    },
+    {
+      check: () => lc.includes("please ensure javascript is enabled"),
+      desc: "requires javascript"
+    },
+    {
+      check: () => lc.includes("enable javascript to view"),
+      desc: "requires javascript"
+    },
+    {
+      check: () => lc.includes("just a moment"),
+      desc: "cloudflare challenge detected in content"
+    },
+    {
+      check: () => lc.includes("verify you are human"),
+      desc: "human verification"
+    },
+    {
+      check: () => lc.includes("captcha required"),
+      desc: "captcha in extracted content"
+    },
+    {
+      check: () => lc.includes("access denied"),
+      desc: "access denied in content"
+    },
+    {
+      check: () => /^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(markdown),
+      desc: "login form only"
+    }
+  ];
+  for (const { check, desc } of suspiciousChecks) {
+    if (check()) {
+      return { ok: false, reason: desc };
+    }
+  }
+  if (title.includes("just a moment") || title.includes("checking your browser")) {
+    return { ok: false, reason: "cloudflare challenge page detected in title" };
+  }
+  return { ok: true };
+}
+export function shouldUseBrowser(url) {
+  try {
+    const parsed = new URL(url);
+    const hostname = parsed.hostname.toLowerCase();
+    const pathname = parsed.pathname.toLowerCase();
+    const jsHeavyDomains = [
+      "react.dev",
+      "nextjs.org",
+      "vuejs.org",
+      "angular.io",
+      "svelte.dev",
+      "docs.expo.dev",
+      "tailwindcss.com",
+      "storybook.js.org"
+    ];
+    if (jsHeavyDomains.some((d) => hostname === d || hostname.endsWith(`.${d}`))) {
+      return true;
+    }
+    if (pathname.includes("/playground") || pathname.includes("/demo") || pathname.includes("/app")) {
+      return true;
+    }
+    if (parsed.hash && parsed.hash.length > 1) {
+      return true;
+    }
+    return false;
+  } catch {
+    return false;
+  }
+}