@intlayer/engine 9.5.9 → 9.5.11

package/dist/esm/scan/fetchText.mjs

@@ -0,0 +1,55 @@
+//#region src/scan/fetchText.ts
+const MAX_REDIRECTS = 5;
+/**
+* Read a response body as text, gunzipping it when the payload itself is
+* gzip-compressed (e.g. `sitemap.xml.gz`, served without `Content-Encoding`).
+*/
+const readBodyAsText = async (response) => {
+	const bytes = new Uint8Array(await response.arrayBuffer());
+	if (!(bytes[0] === 31 && bytes[1] === 139)) return new TextDecoder().decode(bytes);
+	const decompressedStream = new Blob([bytes]).stream().pipeThrough(new DecompressionStream("gzip"));
+	return new Response(decompressedStream).text();
+};
+/**
+* GET a URL as text with a timeout, returning `undefined` when the request is
+* blocked by {@link ScanFetchOptions.shouldFetchUrl} or fails at network level.
+* Redirects are followed manually so every hop goes through the guard.
+*/
+const fetchText = async (url, { userAgent, timeoutMs, shouldFetchUrl }) => {
+	const controller = new AbortController();
+	const timer = setTimeout(() => controller.abort(), timeoutMs);
+	try {
+		let currentUrl = url;
+		for (let redirectCount = 0;; redirectCount++) {
+			if (shouldFetchUrl && !await shouldFetchUrl(currentUrl)) return;
+			const response = await fetch(currentUrl, {
+				headers: {
+					"User-Agent": userAgent,
+					"Accept-Language": "en-US,en;q=0.9"
+				},
+				redirect: "manual",
+				signal: controller.signal
+			});
+			const location = response.headers.get("location");
+			if (response.status >= 300 && response.status < 400 && location && redirectCount < MAX_REDIRECTS) {
+				currentUrl = new URL(location, currentUrl).href;
+				continue;
+			}
+			return {
+				status: response.status,
+				ok: response.ok,
+				text: await readBodyAsText(response),
+				finalUrl: currentUrl,
+				redirected: redirectCount > 0
+			};
+		}
+	} catch {
+		return;
+	} finally {
+		clearTimeout(timer);
+	}
+};
+
+//#endregion
+export { fetchText };
+//# sourceMappingURL=fetchText.mjs.map

package/dist/esm/scan/fetchText.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchText.mjs","names":[],"sources":["../../../src/scan/fetchText.ts"],"sourcesContent":["/** Options shared by every network request issued during a scan. */\nexport type ScanFetchOptions = {\n  userAgent: string;\n  timeoutMs: number;\n  /**\n   * Guard called before fetching any URL discovered on the scanned site\n   * (hreflang alternates, sitemap children…). The hosted audit uses it to\n   * block private / reserved addresses (SSRF). Defaults to allowing all.\n   */\n  shouldFetchUrl?: (url: string) => boolean | Promise<boolean>;\n};\n\n/** Response of {@link fetchText}. */\nexport type FetchTextResult = {\n  status: number;\n  ok: boolean;\n  text: string;\n  /** URL after redirects. */\n  finalUrl: string;\n  /** Whether at least one redirect was followed. */\n  redirected: boolean;\n};\n\nconst MAX_REDIRECTS = 5;\n\n/**\n * Read a response body as text, gunzipping it when the payload itself is\n * gzip-compressed (e.g. `sitemap.xml.gz`, served without `Content-Encoding`).\n */\nconst readBodyAsText = async (response: Response): Promise<string> => {\n  const bytes = new Uint8Array(await response.arrayBuffer());\n  const isGzip = bytes[0] === 0x1f && bytes[1] === 0x8b;\n  if (!isGzip) return new TextDecoder().decode(bytes);\n\n  const decompressedStream = new Blob([bytes])\n    .stream()\n    .pipeThrough(new DecompressionStream('gzip'));\n  return new Response(decompressedStream).text();\n};\n\n/**\n * GET a URL as text with a timeout, returning `undefined` when the request is\n * blocked by {@link ScanFetchOptions.shouldFetchUrl} or fails at network level.\n * Redirects are followed manually so every hop goes through the guard.\n */\nexport const fetchText = async (\n  url: string,\n  { userAgent, timeoutMs, shouldFetchUrl }: ScanFetchOptions\n): Promise<FetchTextResult | undefined> => {\n  const controller = new AbortController();\n  const timer = setTimeout(() => controller.abort(), timeoutMs);\n  try {\n    let currentUrl = url;\n    for (let redirectCount = 0; ; redirectCount++) {\n      if (shouldFetchUrl && !(await shouldFetchUrl(currentUrl))) {\n        return undefined;\n      }\n      const response = await fetch(currentUrl, {\n        headers: {\n          'User-Agent': userAgent,\n          'Accept-Language': 'en-US,en;q=0.9',\n        },\n        redirect: 'manual',\n        signal: controller.signal,\n      });\n      const location = response.headers.get('location');\n      if (\n        response.status >= 300 &&\n        response.status < 400 &&\n        location &&\n        redirectCount < MAX_REDIRECTS\n      ) {\n        currentUrl = new URL(location, currentUrl).href;\n        continue;\n      }\n      return {\n        status: response.status,\n        ok: response.ok,\n        text: await readBodyAsText(response),\n        finalUrl: currentUrl,\n        redirected: redirectCount > 0,\n      };\n    }\n  } catch {\n    return undefined;\n  } finally {\n    clearTimeout(timer);\n  }\n};\n"],"mappings":";AAuBA,MAAM,gBAAgB;;;;;AAMtB,MAAM,iBAAiB,OAAO,aAAwC;CACpE,MAAM,QAAQ,IAAI,WAAW,MAAM,SAAS,YAAY,CAAC;CAEzD,IAAI,EADW,MAAM,OAAO,MAAQ,MAAM,OAAO,MACpC,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK;CAElD,MAAM,qBAAqB,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CACzC,OAAO,CAAC,CACR,YAAY,IAAI,oBAAoB,MAAM,CAAC;CAC9C,OAAO,IAAI,SAAS,kBAAkB,CAAC,CAAC,KAAK;AAC/C;;;;;;AAOA,MAAa,YAAY,OACvB,KACA,EAAE,WAAW,WAAW,qBACiB;CACzC,MAAM,aAAa,IAAI,gBAAgB;CACvC,MAAM,QAAQ,iBAAiB,WAAW,MAAM,GAAG,SAAS;CAC5D,IAAI;EACF,IAAI,aAAa;EACjB,KAAK,IAAI,gBAAgB,IAAK,iBAAiB;GAC7C,IAAI,kBAAkB,CAAE,MAAM,eAAe,UAAU,GACrD;GAEF,MAAM,WAAW,MAAM,MAAM,YAAY;IACvC,SAAS;KACP,cAAc;KACd,mBAAmB;IACrB;IACA,UAAU;IACV,QAAQ,WAAW;GACrB,CAAC;GACD,MAAM,WAAW,SAAS,QAAQ,IAAI,UAAU;GAChD,IACE,SAAS,UAAU,OACnB,SAAS,SAAS,OAClB,YACA,gBAAgB,eAChB;IACA,aAAa,IAAI,IAAI,UAAU,UAAU,CAAC,CAAC;IAC3C;GACF;GACA,OAAO;IACL,QAAQ,SAAS;IACjB,IAAI,SAAS;IACb,MAAM,MAAM,eAAe,QAAQ;IACnC,UAAU;IACV,YAAY,gBAAgB;GAC9B;EACF;CACF,QAAQ;EACN;CACF,UAAU;EACR,aAAa,KAAK;CACpB;AACF"}

package/dist/esm/scan/index.mjs

@@ -1,7 +1,19 @@
+import { fetchText } from "./fetchText.mjs";
 import { mutateScore, scoreRecord, toScorePercent } from "./calculateScore.mjs";
-import { byteLength, extractAnchors, extractHreflangs, extractHtmlDir, extractHtmlLang, extractMetaDescription, extractOgImage, extractScriptUrls, extractTitle, extractVisibleTextStrings, hasCanonical } from "./parseHtml.mjs";
+import { technologySignatures } from "./detection/technologySignatures.mjs";
+import { getFirstPathSegment, getSiteDomain, isAbsoluteUrl, normalizeHostname, normalizeUrl, parseUrl } from "./detection/url.mjs";
+import { detectTechnologies, getTechnologyGlobalNames } from "./detection/detectTechnologies.mjs";
+import { byteLength, extractAnchors, extractCanonicalHref, extractHreflangs, extractHtmlDir, extractHtmlLang, extractMetaDescription, extractOgImage, extractOgLocale, extractOgLocaleAlternates, extractResourceUrls, extractScriptUrls, extractTitle, extractVisibleTextStrings, hasCanonical } from "./parseHtml.mjs";
 import { analyzeBundleContent } from "./analyzeBundleContent.mjs";
-import { checkBundleContent, checkCanonical, checkHtmlAttributes, checkLinguisticStructure, checkRobots, checkSitemap, checkUrlStructure, formatSize } from "./checks.mjs";
+import { findMatchingLocale, getLanguageCode, isRightToLeftLocale, isSameLanguage, isSameLocale, isValidLocaleCode, isValidOpenGraphLocale, looksLikeLocaleCode, normalizeLocaleCode, toOpenGraphLocale } from "./detection/localeCode.mjs";
+import { detectRoutingStrategy, getUrlLocale } from "./detection/detectRoutingStrategy.mjs";
+import { classifyInternalLinks } from "./detection/classifyInternalLinks.mjs";
+import { isPathAllowedByRobots, parseRobots } from "./robots.mjs";
+import { collectSitemapEntries, discoverSitemapUrls, getDefaultSitemapUrls, parseSitemap } from "./sitemap.mjs";
+import { checkBundleContent, checkCanonical, checkHreflang, checkHreflangReciprocity, checkHtmlDir, checkHtmlLang, checkInternalLinks, checkLocaleConsistency, checkOgLocale, checkRobots, checkSitemap, checkXDefault, formatBundleAnalysis, formatSize } from "./checks.mjs";
+import { runScanChecks } from "./runScanChecks.mjs";
 import { scanWebsite } from "./scanWebsite.mjs";
+import { getCheckDetailLines, splitCheckDetails } from "./detection/checkDetails.mjs";
+import { getLocalizedPages, isBaseLocalePage } from "./detection/localizedPages.mjs";
 
-export { analyzeBundleContent, byteLength, checkBundleContent, checkCanonical, checkHtmlAttributes, checkLinguisticStructure, checkRobots, checkSitemap, checkUrlStructure, extractAnchors, extractHreflangs, extractHtmlDir, extractHtmlLang, extractMetaDescription, extractOgImage, extractScriptUrls, extractTitle, extractVisibleTextStrings, formatSize, hasCanonical, mutateScore, scanWebsite, scoreRecord, toScorePercent };
+export { analyzeBundleContent, byteLength, checkBundleContent, checkCanonical, checkHreflang, checkHreflangReciprocity, checkHtmlDir, checkHtmlLang, checkInternalLinks, checkLocaleConsistency, checkOgLocale, checkRobots, checkSitemap, checkXDefault, classifyInternalLinks, collectSitemapEntries, detectRoutingStrategy, detectTechnologies, discoverSitemapUrls, extractAnchors, extractCanonicalHref, extractHreflangs, extractHtmlDir, extractHtmlLang, extractMetaDescription, extractOgImage, extractOgLocale, extractOgLocaleAlternates, extractResourceUrls, extractScriptUrls, extractTitle, extractVisibleTextStrings, fetchText, findMatchingLocale, formatBundleAnalysis, formatSize, getCheckDetailLines, getDefaultSitemapUrls, getFirstPathSegment, getLanguageCode, getLocalizedPages, getSiteDomain, getTechnologyGlobalNames, getUrlLocale, hasCanonical, isAbsoluteUrl, isBaseLocalePage, isPathAllowedByRobots, isRightToLeftLocale, isSameLanguage, isSameLocale, isValidLocaleCode, isValidOpenGraphLocale, looksLikeLocaleCode, mutateScore, normalizeHostname, normalizeLocaleCode, normalizeUrl, parseRobots, parseSitemap, parseUrl, runScanChecks, scanWebsite, scoreRecord, splitCheckDetails, technologySignatures, toOpenGraphLocale, toScorePercent };

package/dist/esm/scan/parseHtml.mjs

@@ -18,17 +18,17 @@ const readAttribute = (attributes, attributeName) => {
 /** Extract the `lang` attribute of the `<html>` element, if present. */
 const extractHtmlLang = (html) => {
 	const htmlTag = html.match(/<html\b([^>]*)>/i);
-	return htmlTag ? readAttribute(htmlTag[1], "lang") : void 0;
+	return htmlTag?.[1] ? readAttribute(htmlTag[1], "lang") : void 0;
 };
 /** Extract the `dir` attribute of the `<html>` element, if present. */
 const extractHtmlDir = (html) => {
 	const htmlTag = html.match(/<html\b([^>]*)>/i);
-	return htmlTag ? readAttribute(htmlTag[1], "dir") : void 0;
+	return htmlTag?.[1] ? readAttribute(htmlTag[1], "dir") : void 0;
 };
 /** Extract the document `<title>` text. */
 const extractTitle = (html) => {
 	const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-	return match ? match[1].trim() : "";
+	return match?.[1] ? match[1].trim() : "";
 };
 /** Extract the `<meta name="description">` content. */
 const extractMetaDescription = (html) => {
@@ -39,12 +39,28 @@ const extractMetaDescription = (html) => {
 /** Extract the `<meta property="og:image">` content. */
 const extractOgImage = (html) => {
 	const metas = html.match(/<meta\b[^>]*>/gi) ?? [];
-	for (const meta of metas) if (/property\s*=\s*("|')?og:image\1?/i.test(meta)) return readAttribute(meta, "content");
+	for (const meta of metas) if (/property\s*=\s*("|')?og:image\1?[\s>/]/i.test(meta)) return readAttribute(meta, "content");
 };
-/** Whether a `<link rel="canonical">` element is present. */
-const hasCanonical = (html) => {
-	return (html.match(/<link\b[^>]*>/gi) ?? []).some((link) => /rel\s*=\s*("|')?canonical\1?/i.test(link));
+/** Extract the `<meta property="og:locale">` content. */
+const extractOgLocale = (html) => {
+	const metas = html.match(/<meta\b[^>]*>/gi) ?? [];
+	for (const meta of metas) if (/property\s*=\s*("|')?og:locale\1?[\s>/]/i.test(meta)) return readAttribute(meta, "content");
+};
+/** Extract every `<meta property="og:locale:alternate">` content. */
+const extractOgLocaleAlternates = (html) => {
+	return (html.match(/<meta\b[^>]*>/gi) ?? []).flatMap((meta) => {
+		if (!/property\s*=\s*("|')?og:locale:alternate\1?[\s>/]/i.test(meta)) return [];
+		const content = readAttribute(meta, "content");
+		return content ? [content] : [];
+	});
+};
+/** Extract the `href` of the `<link rel="canonical">` element, if present. */
+const extractCanonicalHref = (html) => {
+	const canonicalLink = (html.match(/<link\b[^>]*>/gi) ?? []).find((link) => /rel\s*=\s*("|')?canonical\1?/i.test(link));
+	return canonicalLink ? readAttribute(canonicalLink, "href") ?? "" : void 0;
 };
+/** Whether a `<link rel="canonical">` element is present. */
+const hasCanonical = (html) => extractCanonicalHref(html) !== void 0;
 /** Extract every `<link rel="alternate" hreflang="…" href="…">` element. */
 const extractHreflangs = (html) => {
 	const links = html.match(/<link\b[^>]*>/gi) ?? [];
@@ -90,12 +106,15 @@ const extractAnchors = (html) => {
 	const anchorPattern = /<a\b([^>]*)>([\s\S]*?)<\/a>/gi;
 	let match = anchorPattern.exec(html);
 	while (match !== null) {
-		const href = readAttribute(match[1], "href");
+		const rawAttrs = match[1] ?? "";
+		const rawContent = match[2] ?? "";
+		const href = readAttribute(rawAttrs, "href");
 		if (href) {
-			const text = match[2].replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+			const text = rawContent.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
 			anchors.push({
 				href,
-				text
+				text,
+				hreflang: readAttribute(rawAttrs, "hreflang")
 			});
 		}
 		match = anchorPattern.exec(html);
@@ -103,6 +122,28 @@ const extractAnchors = (html) => {
 	return anchors;
 };
 /**
+* Extract every resource URL referenced by the document (`<script src>`,
+* `<link href>`, `<iframe src>`), used to fingerprint third-party services.
+*
+* @param html - The raw HTML document.
+* @param baseUrl - Base URL used to resolve relative URLs.
+* @returns Absolute, de-duplicated URLs.
+*/
+const extractResourceUrls = (html, baseUrl) => {
+	const urls = /* @__PURE__ */ new Set();
+	const tagPattern = /<(script|link|iframe)\b([^>]*)>/gi;
+	let match = tagPattern.exec(html);
+	while (match !== null) {
+		const attributeName = match[1]?.toLowerCase() === "link" ? "href" : "src";
+		const rawUrl = readAttribute(match[2] ?? "", attributeName);
+		if (rawUrl) try {
+			urls.add(new URL(rawUrl, baseUrl).href);
+		} catch {}
+		match = tagPattern.exec(html);
+	}
+	return Array.from(urls);
+};
+/**
 * Extract visible text snippets from an HTML document (scripts, styles and
 * tags stripped). Used to approximate the rendered content size without a DOM.
 */
@@ -111,5 +152,5 @@ const extractVisibleTextStrings = (html) => {
 };
 
 //#endregion
-export { byteLength, extractAnchors, extractHreflangs, extractHtmlDir, extractHtmlLang, extractMetaDescription, extractOgImage, extractScriptUrls, extractTitle, extractVisibleTextStrings, hasCanonical };
+export { byteLength, extractAnchors, extractCanonicalHref, extractHreflangs, extractHtmlDir, extractHtmlLang, extractMetaDescription, extractOgImage, extractOgLocale, extractOgLocaleAlternates, extractResourceUrls, extractScriptUrls, extractTitle, extractVisibleTextStrings, hasCanonical };
 //# sourceMappingURL=parseHtml.mjs.map

package/dist/esm/scan/parseHtml.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"parseHtml.mjs","names":[],"sources":["../../../src/scan/parseHtml.ts"],"sourcesContent":["/**\n * Tiny dependency-free HTML extraction helpers.\n *\n * The hosted backend audit relies on Cheerio + a real browser, but the CLI scan\n * must stay dependency-light. These regex-based helpers cover the handful of\n * head/anchor signals the score needs. They are intentionally forgiving: when a\n * tag can't be parsed it is simply skipped rather than throwing.\n */\n\n/** Compute the UTF-8 byte length of a string in both Node and browser builds. */\nexport const byteLength = (text: string): number =>\n  typeof Buffer !== 'undefined'\n    ? Buffer.byteLength(text, 'utf-8')\n    : new TextEncoder().encode(text).length;\n\n/** Read an attribute value off a single tag's attribute string. */\nconst readAttribute = (\n  attributes: string,\n  attributeName: string\n): string | undefined => {\n  const match = attributes.match(\n    new RegExp(`${attributeName}\\\\s*=\\\\s*(\"([^\"]*)\"|'([^']*)'|([^\\\\s>]+))`, 'i')\n  );\n  if (!match) return undefined;\n  return match[2] ?? match[3] ?? match[4];\n};\n\n/** Extract the `lang` attribute of the `<html>` element, if present. */\nexport const extractHtmlLang = (html: string): string | undefined => {\n  const htmlTag = html.match(/<html\\b([^>]*)>/i);\n  return htmlTag ? readAttribute(htmlTag[1], 'lang') : undefined;\n};\n\n/** Extract the `dir` attribute of the `<html>` element, if present. */\nexport const extractHtmlDir = (html: string): string | undefined => {\n  const htmlTag = html.match(/<html\\b([^>]*)>/i);\n  return htmlTag ? readAttribute(htmlTag[1], 'dir') : undefined;\n};\n\n/** Extract the document `<title>` text. */\nexport const extractTitle = (html: string): string => {\n  const match = html.match(/<title[^>]*>([\\s\\S]*?)<\\/title>/i);\n  return match ? match[1].trim() : '';\n};\n\n/** Extract the `<meta name=\"description\">` content. */\nexport const extractMetaDescription = (html: string): string => {\n  const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n  for (const meta of metas) {\n    if (/name\\s*=\\s*(\"|')?description\\1?/i.test(meta)) {\n      return readAttribute(meta, 'content') ?? '';\n    }\n  }\n  return '';\n};\n\n/** Extract the `<meta property=\"og:image\">` content. */\nexport const extractOgImage = (html: string): string | undefined => {\n  const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n  for (const meta of metas) {\n    if (/property\\s*=\\s*(\"|')?og:image\\1?/i.test(meta)) {\n      return readAttribute(meta, 'content');\n    }\n  }\n  return undefined;\n};\n\n/** Whether a `<link rel=\"canonical\">` element is present. */\nexport const hasCanonical = (html: string): boolean => {\n  const links = html.match(/<link\\b[^>]*>/gi) ?? [];\n  return links.some((link) => /rel\\s*=\\s*(\"|')?canonical\\1?/i.test(link));\n};\n\n/** A parsed `<link rel=\"alternate\" hreflang>` element. */\nexport type HreflangLink = { hreflang: string; href: string };\n\n/** Extract every `<link rel=\"alternate\" hreflang=\"…\" href=\"…\">` element. */\nexport const extractHreflangs = (html: string): HreflangLink[] => {\n  const links = html.match(/<link\\b[^>]*>/gi) ?? [];\n  const result: HreflangLink[] = [];\n  for (const link of links) {\n    if (!/rel\\s*=\\s*(\"|')?alternate\\1?/i.test(link)) continue;\n    const hreflang = readAttribute(link, 'hreflang');\n    const href = readAttribute(link, 'href');\n    if (hreflang && href) result.push({ hreflang, href });\n  }\n  return result;\n};\n\n/**\n * Extract every eagerly-loaded script URL: `<script src>`,\n * `<link rel=\"modulepreload\">` and `<link rel=\"preload\" as=\"script\">`.\n *\n * @param html - The raw HTML document.\n * @param baseUrl - Base URL used to resolve relative script URLs.\n * @returns Absolute, de-duplicated script URLs.\n */\nexport const extractScriptUrls = (html: string, baseUrl: string): string[] => {\n  const urls = new Set<string>();\n\n  const add = (raw: string | undefined) => {\n    if (!raw) return;\n    try {\n      urls.add(new URL(raw, baseUrl).href);\n    } catch {\n      /* ignore malformed URLs */\n    }\n  };\n\n  for (const script of html.match(/<script\\b[^>]*>/gi) ?? []) {\n    add(readAttribute(script, 'src'));\n  }\n\n  for (const link of html.match(/<link\\b[^>]*>/gi) ?? []) {\n    const rel = readAttribute(link, 'rel')?.toLowerCase();\n    const as = readAttribute(link, 'as')?.toLowerCase();\n    if (rel === 'modulepreload' || (rel === 'preload' && as === 'script')) {\n      add(readAttribute(link, 'href'));\n    }\n  }\n\n  return Array.from(urls);\n};\n\n/** A parsed `<a href>` anchor. */\nexport type Anchor = { href: string; text: string };\n\n/** Extract every `<a href=\"…\">text</a>` anchor from the document. */\nexport const extractAnchors = (html: string): Anchor[] => {\n  const anchors: Anchor[] = [];\n  const anchorPattern = /<a\\b([^>]*)>([\\s\\S]*?)<\\/a>/gi;\n  let match = anchorPattern.exec(html);\n  while (match !== null) {\n    const href = readAttribute(match[1], 'href');\n    if (href) {\n      const text = match[2]\n        .replace(/<[^>]+>/g, ' ')\n        .replace(/\\s+/g, ' ')\n        .trim();\n      anchors.push({ href, text });\n    }\n    match = anchorPattern.exec(html);\n  }\n  return anchors;\n};\n\n/**\n * Extract visible text snippets from an HTML document (scripts, styles and\n * tags stripped). Used to approximate the rendered content size without a DOM.\n */\nexport const extractVisibleTextStrings = (html: string): string[] => {\n  const withoutNonVisible = html\n    .replace(/<script[\\s\\S]*?<\\/script>/gi, ' ')\n    .replace(/<style[\\s\\S]*?<\\/style>/gi, ' ')\n    .replace(/<noscript[\\s\\S]*?<\\/noscript>/gi, ' ')\n    .replace(/<!--[\\s\\S]*?-->/g, ' ');\n\n  return withoutNonVisible\n    .replace(/<[^>]+>/g, '\\n')\n    .split('\\n')\n    .map((line) => line.replace(/\\s+/g, ' ').trim())\n    .filter((line) => line.length > 1);\n};\n"],"mappings":";;;;;;;;;;AAUA,MAAa,cAAc,SACzB,OAAO,WAAW,cACd,OAAO,WAAW,MAAM,OAAO,IAC/B,IAAI,YAAY,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;;AAGrC,MAAM,iBACJ,YACA,kBACuB;CACvB,MAAM,QAAQ,WAAW,MACvB,IAAI,OAAO,GAAG,cAAc,4CAA4C,GAAG,CAC7E;CACA,IAAI,CAAC,OAAO,OAAO;CACnB,OAAO,MAAM,MAAM,MAAM,MAAM,MAAM;AACvC;;AAGA,MAAa,mBAAmB,SAAqC;CACnE,MAAM,UAAU,KAAK,MAAM,kBAAkB;CAC7C,OAAO,UAAU,cAAc,QAAQ,IAAI,MAAM,IAAI;AACvD;;AAGA,MAAa,kBAAkB,SAAqC;CAClE,MAAM,UAAU,KAAK,MAAM,kBAAkB;CAC7C,OAAO,UAAU,cAAc,QAAQ,IAAI,KAAK,IAAI;AACtD;;AAGA,MAAa,gBAAgB,SAAyB;CACpD,MAAM,QAAQ,KAAK,MAAM,kCAAkC;CAC3D,OAAO,QAAQ,MAAM,EAAE,CAAC,KAAK,IAAI;AACnC;;AAGA,MAAa,0BAA0B,SAAyB;CAC9D,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,KAAK,MAAM,QAAQ,OACjB,IAAI,mCAAmC,KAAK,IAAI,GAC9C,OAAO,cAAc,MAAM,SAAS,KAAK;CAG7C,OAAO;AACT;;AAGA,MAAa,kBAAkB,SAAqC;CAClE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,KAAK,MAAM,QAAQ,OACjB,IAAI,oCAAoC,KAAK,IAAI,GAC/C,OAAO,cAAc,MAAM,SAAS;AAI1C;;AAGA,MAAa,gBAAgB,SAA0B;CAErD,QADc,KAAK,MAAM,iBAAiB,KAAK,CAAC,EACpC,CAAC,MAAM,SAAS,gCAAgC,KAAK,IAAI,CAAC;AACxE;;AAMA,MAAa,oBAAoB,SAAiC;CAChE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,MAAM,SAAyB,CAAC;CAChC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,gCAAgC,KAAK,IAAI,GAAG;EACjD,MAAM,WAAW,cAAc,MAAM,UAAU;EAC/C,MAAM,OAAO,cAAc,MAAM,MAAM;EACvC,IAAI,YAAY,MAAM,OAAO,KAAK;GAAE;GAAU;EAAK,CAAC;CACtD;CACA,OAAO;AACT;;;;;;;;;AAUA,MAAa,qBAAqB,MAAc,YAA8B;CAC5E,MAAM,uBAAO,IAAI,IAAY;CAE7B,MAAM,OAAO,QAA4B;EACvC,IAAI,CAAC,KAAK;EACV,IAAI;GACF,KAAK,IAAI,IAAI,IAAI,KAAK,OAAO,CAAC,CAAC,IAAI;EACrC,QAAQ,CAER;CACF;CAEA,KAAK,MAAM,UAAU,KAAK,MAAM,mBAAmB,KAAK,CAAC,GACvD,IAAI,cAAc,QAAQ,KAAK,CAAC;CAGlC,KAAK,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC,GAAG;EACtD,MAAM,MAAM,cAAc,MAAM,KAAK,CAAC,EAAE,YAAY;EACpD,MAAM,KAAK,cAAc,MAAM,IAAI,CAAC,EAAE,YAAY;EAClD,IAAI,QAAQ,mBAAoB,QAAQ,aAAa,OAAO,UAC1D,IAAI,cAAc,MAAM,MAAM,CAAC;CAEnC;CAEA,OAAO,MAAM,KAAK,IAAI;AACxB;;AAMA,MAAa,kBAAkB,SAA2B;CACxD,MAAM,UAAoB,CAAC;CAC3B,MAAM,gBAAgB;CACtB,IAAI,QAAQ,cAAc,KAAK,IAAI;CACnC,OAAO,UAAU,MAAM;EACrB,MAAM,OAAO,cAAc,MAAM,IAAI,MAAM;EAC3C,IAAI,MAAM;GACR,MAAM,OAAO,MAAM,EAAE,CAClB,QAAQ,YAAY,GAAG,CAAC,CACxB,QAAQ,QAAQ,GAAG,CAAC,CACpB,KAAK;GACR,QAAQ,KAAK;IAAE;IAAM;GAAK,CAAC;EAC7B;EACA,QAAQ,cAAc,KAAK,IAAI;CACjC;CACA,OAAO;AACT;;;;;AAMA,MAAa,6BAA6B,SAA2B;CAOnE,OAN0B,KACvB,QAAQ,+BAA+B,GAAG,CAAC,CAC3C,QAAQ,6BAA6B,GAAG,CAAC,CACzC,QAAQ,mCAAmC,GAAG,CAAC,CAC/C,QAAQ,oBAAoB,GAER,CAAC,CACrB,QAAQ,YAAY,IAAI,CAAC,CACzB,MAAM,IAAI,CAAC,CACX,KAAK,SAAS,KAAK,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAC/C,QAAQ,SAAS,KAAK,SAAS,CAAC;AACrC"}
+{"version":3,"file":"parseHtml.mjs","names":[],"sources":["../../../src/scan/parseHtml.ts"],"sourcesContent":["/**\n * Tiny dependency-free HTML extraction helpers.\n *\n * The hosted backend audit relies on Cheerio + a real browser, but the CLI scan\n * must stay dependency-light. These regex-based helpers cover the handful of\n * head/anchor signals the score needs. They are intentionally forgiving: when a\n * tag can't be parsed it is simply skipped rather than throwing.\n */\n\n/** Compute the UTF-8 byte length of a string in both Node and browser builds. */\nexport const byteLength = (text: string): number =>\n  typeof Buffer !== 'undefined'\n    ? Buffer.byteLength(text, 'utf-8')\n    : new TextEncoder().encode(text).length;\n\n/** Read an attribute value off a single tag's attribute string. */\nconst readAttribute = (\n  attributes: string,\n  attributeName: string\n): string | undefined => {\n  const match = attributes.match(\n    new RegExp(`${attributeName}\\\\s*=\\\\s*(\"([^\"]*)\"|'([^']*)'|([^\\\\s>]+))`, 'i')\n  );\n  if (!match) return undefined;\n  return match[2] ?? match[3] ?? match[4];\n};\n\n/** Extract the `lang` attribute of the `<html>` element, if present. */\nexport const extractHtmlLang = (html: string): string | undefined => {\n  const htmlTag = html.match(/<html\\b([^>]*)>/i);\n  return htmlTag?.[1] ? readAttribute(htmlTag[1], 'lang') : undefined;\n};\n\n/** Extract the `dir` attribute of the `<html>` element, if present. */\nexport const extractHtmlDir = (html: string): string | undefined => {\n  const htmlTag = html.match(/<html\\b([^>]*)>/i);\n  return htmlTag?.[1] ? readAttribute(htmlTag[1], 'dir') : undefined;\n};\n\n/** Extract the document `<title>` text. */\nexport const extractTitle = (html: string): string => {\n  const match = html.match(/<title[^>]*>([\\s\\S]*?)<\\/title>/i);\n  return match?.[1] ? match[1].trim() : '';\n};\n\n/** Extract the `<meta name=\"description\">` content. */\nexport const extractMetaDescription = (html: string): string => {\n  const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n  for (const meta of metas) {\n    if (/name\\s*=\\s*(\"|')?description\\1?/i.test(meta)) {\n      return readAttribute(meta, 'content') ?? '';\n    }\n  }\n  return '';\n};\n\n/** Extract the `<meta property=\"og:image\">` content. */\nexport const extractOgImage = (html: string): string | undefined => {\n  const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n  for (const meta of metas) {\n    if (/property\\s*=\\s*(\"|')?og:image\\1?[\\s>/]/i.test(meta)) {\n      return readAttribute(meta, 'content');\n    }\n  }\n  return undefined;\n};\n\n/** Extract the `<meta property=\"og:locale\">` content. */\nexport const extractOgLocale = (html: string): string | undefined => {\n  const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n  for (const meta of metas) {\n    if (/property\\s*=\\s*(\"|')?og:locale\\1?[\\s>/]/i.test(meta)) {\n      return readAttribute(meta, 'content');\n    }\n  }\n  return undefined;\n};\n\n/** Extract every `<meta property=\"og:locale:alternate\">` content. */\nexport const extractOgLocaleAlternates = (html: string): string[] => {\n  const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n  return metas.flatMap((meta) => {\n    if (!/property\\s*=\\s*(\"|')?og:locale:alternate\\1?[\\s>/]/i.test(meta)) {\n      return [];\n    }\n    const content = readAttribute(meta, 'content');\n    return content ? [content] : [];\n  });\n};\n\n/** Extract the `href` of the `<link rel=\"canonical\">` element, if present. */\nexport const extractCanonicalHref = (html: string): string | undefined => {\n  const links = html.match(/<link\\b[^>]*>/gi) ?? [];\n  const canonicalLink = links.find((link) =>\n    /rel\\s*=\\s*(\"|')?canonical\\1?/i.test(link)\n  );\n  return canonicalLink\n    ? (readAttribute(canonicalLink, 'href') ?? '')\n    : undefined;\n};\n\n/** Whether a `<link rel=\"canonical\">` element is present. */\nexport const hasCanonical = (html: string): boolean =>\n  extractCanonicalHref(html) !== undefined;\n\n/** A parsed `<link rel=\"alternate\" hreflang>` element. */\nexport type HreflangLink = { hreflang: string; href: string };\n\n/** Extract every `<link rel=\"alternate\" hreflang=\"…\" href=\"…\">` element. */\nexport const extractHreflangs = (html: string): HreflangLink[] => {\n  const links = html.match(/<link\\b[^>]*>/gi) ?? [];\n  const result: HreflangLink[] = [];\n  for (const link of links) {\n    if (!/rel\\s*=\\s*(\"|')?alternate\\1?/i.test(link)) continue;\n    const hreflang = readAttribute(link, 'hreflang');\n    const href = readAttribute(link, 'href');\n    if (hreflang && href) result.push({ hreflang, href });\n  }\n  return result;\n};\n\n/**\n * Extract every eagerly-loaded script URL: `<script src>`,\n * `<link rel=\"modulepreload\">` and `<link rel=\"preload\" as=\"script\">`.\n *\n * @param html - The raw HTML document.\n * @param baseUrl - Base URL used to resolve relative script URLs.\n * @returns Absolute, de-duplicated script URLs.\n */\nexport const extractScriptUrls = (html: string, baseUrl: string): string[] => {\n  const urls = new Set<string>();\n\n  const add = (raw: string | undefined) => {\n    if (!raw) return;\n    try {\n      urls.add(new URL(raw, baseUrl).href);\n    } catch {\n      /* ignore malformed URLs */\n    }\n  };\n\n  for (const script of html.match(/<script\\b[^>]*>/gi) ?? []) {\n    add(readAttribute(script, 'src'));\n  }\n\n  for (const link of html.match(/<link\\b[^>]*>/gi) ?? []) {\n    const rel = readAttribute(link, 'rel')?.toLowerCase();\n    const as = readAttribute(link, 'as')?.toLowerCase();\n    if (rel === 'modulepreload' || (rel === 'preload' && as === 'script')) {\n      add(readAttribute(link, 'href'));\n    }\n  }\n\n  return Array.from(urls);\n};\n\n/** A parsed `<a href>` anchor. */\nexport type Anchor = {\n  href: string;\n  text: string;\n  /** `hreflang` attribute, set on language-switcher links. */\n  hreflang?: string;\n};\n\n/** Extract every `<a href=\"…\">text</a>` anchor from the document. */\nexport const extractAnchors = (html: string): Anchor[] => {\n  const anchors: Anchor[] = [];\n  const anchorPattern = /<a\\b([^>]*)>([\\s\\S]*?)<\\/a>/gi;\n  let match = anchorPattern.exec(html);\n  while (match !== null) {\n    const rawAttrs = match[1] ?? '';\n    const rawContent = match[2] ?? '';\n    const href = readAttribute(rawAttrs, 'href');\n    if (href) {\n      const text = rawContent\n        .replace(/<[^>]+>/g, ' ')\n        .replace(/\\s+/g, ' ')\n        .trim();\n      anchors.push({\n        href,\n        text,\n        hreflang: readAttribute(rawAttrs, 'hreflang'),\n      });\n    }\n    match = anchorPattern.exec(html);\n  }\n  return anchors;\n};\n\n/**\n * Extract every resource URL referenced by the document (`<script src>`,\n * `<link href>`, `<iframe src>`), used to fingerprint third-party services.\n *\n * @param html - The raw HTML document.\n * @param baseUrl - Base URL used to resolve relative URLs.\n * @returns Absolute, de-duplicated URLs.\n */\nexport const extractResourceUrls = (\n  html: string,\n  baseUrl: string\n): string[] => {\n  const urls = new Set<string>();\n  const tagPattern = /<(script|link|iframe)\\b([^>]*)>/gi;\n  let match = tagPattern.exec(html);\n  while (match !== null) {\n    const attributeName = match[1]?.toLowerCase() === 'link' ? 'href' : 'src';\n    const rawUrl = readAttribute(match[2] ?? '', attributeName);\n    if (rawUrl) {\n      try {\n        urls.add(new URL(rawUrl, baseUrl).href);\n      } catch {\n        /* ignore malformed URLs */\n      }\n    }\n    match = tagPattern.exec(html);\n  }\n  return Array.from(urls);\n};\n\n/**\n * Extract visible text snippets from an HTML document (scripts, styles and\n * tags stripped). Used to approximate the rendered content size without a DOM.\n */\nexport const extractVisibleTextStrings = (html: string): string[] => {\n  const withoutNonVisible = html\n    .replace(/<script[\\s\\S]*?<\\/script>/gi, ' ')\n    .replace(/<style[\\s\\S]*?<\\/style>/gi, ' ')\n    .replace(/<noscript[\\s\\S]*?<\\/noscript>/gi, ' ')\n    .replace(/<!--[\\s\\S]*?-->/g, ' ');\n\n  return withoutNonVisible\n    .replace(/<[^>]+>/g, '\\n')\n    .split('\\n')\n    .map((line) => line.replace(/\\s+/g, ' ').trim())\n    .filter((line) => line.length > 1);\n};\n"],"mappings":";;;;;;;;;;AAUA,MAAa,cAAc,SACzB,OAAO,WAAW,cACd,OAAO,WAAW,MAAM,OAAO,IAC/B,IAAI,YAAY,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;;AAGrC,MAAM,iBACJ,YACA,kBACuB;CACvB,MAAM,QAAQ,WAAW,MACvB,IAAI,OAAO,GAAG,cAAc,4CAA4C,GAAG,CAC7E;CACA,IAAI,CAAC,OAAO,OAAO;CACnB,OAAO,MAAM,MAAM,MAAM,MAAM,MAAM;AACvC;;AAGA,MAAa,mBAAmB,SAAqC;CACnE,MAAM,UAAU,KAAK,MAAM,kBAAkB;CAC7C,OAAO,UAAU,KAAK,cAAc,QAAQ,IAAI,MAAM,IAAI;AAC5D;;AAGA,MAAa,kBAAkB,SAAqC;CAClE,MAAM,UAAU,KAAK,MAAM,kBAAkB;CAC7C,OAAO,UAAU,KAAK,cAAc,QAAQ,IAAI,KAAK,IAAI;AAC3D;;AAGA,MAAa,gBAAgB,SAAyB;CACpD,MAAM,QAAQ,KAAK,MAAM,kCAAkC;CAC3D,OAAO,QAAQ,KAAK,MAAM,EAAE,CAAC,KAAK,IAAI;AACxC;;AAGA,MAAa,0BAA0B,SAAyB;CAC9D,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,KAAK,MAAM,QAAQ,OACjB,IAAI,mCAAmC,KAAK,IAAI,GAC9C,OAAO,cAAc,MAAM,SAAS,KAAK;CAG7C,OAAO;AACT;;AAGA,MAAa,kBAAkB,SAAqC;CAClE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,KAAK,MAAM,QAAQ,OACjB,IAAI,0CAA0C,KAAK,IAAI,GACrD,OAAO,cAAc,MAAM,SAAS;AAI1C;;AAGA,MAAa,mBAAmB,SAAqC;CACnE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,KAAK,MAAM,QAAQ,OACjB,IAAI,2CAA2C,KAAK,IAAI,GACtD,OAAO,cAAc,MAAM,SAAS;AAI1C;;AAGA,MAAa,6BAA6B,SAA2B;CAEnE,QADc,KAAK,MAAM,iBAAiB,KAAK,CAAC,EACpC,CAAC,SAAS,SAAS;EAC7B,IAAI,CAAC,qDAAqD,KAAK,IAAI,GACjE,OAAO,CAAC;EAEV,MAAM,UAAU,cAAc,MAAM,SAAS;EAC7C,OAAO,UAAU,CAAC,OAAO,IAAI,CAAC;CAChC,CAAC;AACH;;AAGA,MAAa,wBAAwB,SAAqC;CAExE,MAAM,iBADQ,KAAK,MAAM,iBAAiB,KAAK,CAAC,EACrB,CAAC,MAAM,SAChC,gCAAgC,KAAK,IAAI,CAC3C;CACA,OAAO,gBACF,cAAc,eAAe,MAAM,KAAK,KACzC;AACN;;AAGA,MAAa,gBAAgB,SAC3B,qBAAqB,IAAI,MAAM;;AAMjC,MAAa,oBAAoB,SAAiC;CAChE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,MAAM,SAAyB,CAAC;CAChC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,gCAAgC,KAAK,IAAI,GAAG;EACjD,MAAM,WAAW,cAAc,MAAM,UAAU;EAC/C,MAAM,OAAO,cAAc,MAAM,MAAM;EACvC,IAAI,YAAY,MAAM,OAAO,KAAK;GAAE;GAAU;EAAK,CAAC;CACtD;CACA,OAAO;AACT;;;;;;;;;AAUA,MAAa,qBAAqB,MAAc,YAA8B;CAC5E,MAAM,uBAAO,IAAI,IAAY;CAE7B,MAAM,OAAO,QAA4B;EACvC,IAAI,CAAC,KAAK;EACV,IAAI;GACF,KAAK,IAAI,IAAI,IAAI,KAAK,OAAO,CAAC,CAAC,IAAI;EACrC,QAAQ,CAER;CACF;CAEA,KAAK,MAAM,UAAU,KAAK,MAAM,mBAAmB,KAAK,CAAC,GACvD,IAAI,cAAc,QAAQ,KAAK,CAAC;CAGlC,KAAK,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC,GAAG;EACtD,MAAM,MAAM,cAAc,MAAM,KAAK,CAAC,EAAE,YAAY;EACpD,MAAM,KAAK,cAAc,MAAM,IAAI,CAAC,EAAE,YAAY;EAClD,IAAI,QAAQ,mBAAoB,QAAQ,aAAa,OAAO,UAC1D,IAAI,cAAc,MAAM,MAAM,CAAC;CAEnC;CAEA,OAAO,MAAM,KAAK,IAAI;AACxB;;AAWA,MAAa,kBAAkB,SAA2B;CACxD,MAAM,UAAoB,CAAC;CAC3B,MAAM,gBAAgB;CACtB,IAAI,QAAQ,cAAc,KAAK,IAAI;CACnC,OAAO,UAAU,MAAM;EACrB,MAAM,WAAW,MAAM,MAAM;EAC7B,MAAM,aAAa,MAAM,MAAM;EAC/B,MAAM,OAAO,cAAc,UAAU,MAAM;EAC3C,IAAI,MAAM;GACR,MAAM,OAAO,WACV,QAAQ,YAAY,GAAG,CAAC,CACxB,QAAQ,QAAQ,GAAG,CAAC,CACpB,KAAK;GACR,QAAQ,KAAK;IACX;IACA;IACA,UAAU,cAAc,UAAU,UAAU;GAC9C,CAAC;EACH;EACA,QAAQ,cAAc,KAAK,IAAI;CACjC;CACA,OAAO;AACT;;;;;;;;;AAUA,MAAa,uBACX,MACA,YACa;CACb,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,aAAa;CACnB,IAAI,QAAQ,WAAW,KAAK,IAAI;CAChC,OAAO,UAAU,MAAM;EACrB,MAAM,gBAAgB,MAAM,EAAE,EAAE,YAAY,MAAM,SAAS,SAAS;EACpE,MAAM,SAAS,cAAc,MAAM,MAAM,IAAI,aAAa;EAC1D,IAAI,QACF,IAAI;GACF,KAAK,IAAI,IAAI,IAAI,QAAQ,OAAO,CAAC,CAAC,IAAI;EACxC,QAAQ,CAER;EAEF,QAAQ,WAAW,KAAK,IAAI;CAC9B;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;;;;;AAMA,MAAa,6BAA6B,SAA2B;CAOnE,OAN0B,KACvB,QAAQ,+BAA+B,GAAG,CAAC,CAC3C,QAAQ,6BAA6B,GAAG,CAAC,CACzC,QAAQ,mCAAmC,GAAG,CAAC,CAC/C,QAAQ,oBAAoB,GAER,CAAC,CACrB,QAAQ,YAAY,IAAI,CAAC,CACzB,MAAM,IAAI,CAAC,CACX,KAAK,SAAS,KAAK,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAC/C,QAAQ,SAAS,KAAK,SAAS,CAAC;AACrC"}

package/dist/esm/scan/robots.mjs

@@ -0,0 +1,63 @@
+//#region src/scan/robots.ts
+/**
+* Parse a `robots.txt`, keeping the rules Googlebot would follow: the
+* `googlebot` group when one exists, the `*` group otherwise.
+*/
+const parseRobots = (content) => {
+	const groups = [];
+	const sitemapUrls = [];
+	let currentGroup;
+	let isReadingUserAgents = false;
+	for (const rawLine of content.split(/\r?\n/)) {
+		const line = rawLine.replace(/#.*$/, "").trim();
+		const separatorIndex = line.indexOf(":");
+		if (separatorIndex === -1) continue;
+		const directive = line.slice(0, separatorIndex).trim().toLowerCase();
+		const value = line.slice(separatorIndex + 1).trim();
+		if (directive === "sitemap") {
+			if (value) sitemapUrls.push(value);
+			continue;
+		}
+		if (directive === "user-agent") {
+			if (!isReadingUserAgents || !currentGroup) {
+				currentGroup = {
+					userAgents: [],
+					disallowedPaths: [],
+					allowedPaths: []
+				};
+				groups.push(currentGroup);
+			}
+			currentGroup.userAgents.push(value.toLowerCase());
+			isReadingUserAgents = true;
+			continue;
+		}
+		isReadingUserAgents = false;
+		if (!currentGroup || !value) continue;
+		if (directive === "disallow") currentGroup.disallowedPaths.push(value);
+		if (directive === "allow") currentGroup.allowedPaths.push(value);
+	}
+	const googlebotGroups = groups.filter(({ userAgents }) => userAgents.includes("googlebot"));
+	const applicableGroups = googlebotGroups.length > 0 ? googlebotGroups : groups.filter(({ userAgents }) => userAgents.includes("*"));
+	return {
+		disallowedPaths: applicableGroups.flatMap(({ disallowedPaths }) => disallowedPaths),
+		allowedPaths: applicableGroups.flatMap(({ allowedPaths }) => allowedPaths),
+		sitemapUrls
+	};
+};
+/** Convert a robots.txt path pattern (`*`, `$`) into a RegExp. */
+const robotsPatternToRegExp = (pattern) => new RegExp(`^${pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\\\$$/, "$")}`);
+/**
+* Whether Googlebot may crawl `pathWithSearch` (path + query string) under the
+* parsed rules. Follows Google's precedence: the longest matching rule wins,
+* `Allow` wins ties.
+*/
+const isPathAllowedByRobots = (pathWithSearch, robots) => {
+	const longestMatch = (patterns) => patterns.reduce((longest, pattern) => robotsPatternToRegExp(pattern).test(pathWithSearch) ? Math.max(longest, pattern.length) : longest, -1);
+	const disallowLength = longestMatch(robots.disallowedPaths);
+	if (disallowLength === -1) return true;
+	return longestMatch(robots.allowedPaths) >= disallowLength;
+};
+
+//#endregion
+export { isPathAllowedByRobots, parseRobots };
+//# sourceMappingURL=robots.mjs.map

package/dist/esm/scan/robots.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.mjs","names":[],"sources":["../../../src/scan/robots.ts"],"sourcesContent":["/** Rules of a `robots.txt` that apply to Google's crawler. */\nexport type ParsedRobots = {\n  /** `Disallow` paths of the group applying to Googlebot (or `*`). */\n  disallowedPaths: string[];\n  /** `Allow` paths of the same group. */\n  allowedPaths: string[];\n  /** Absolute URLs declared with `Sitemap:` directives. */\n  sitemapUrls: string[];\n};\n\ntype RobotsGroup = {\n  userAgents: string[];\n  disallowedPaths: string[];\n  allowedPaths: string[];\n};\n\n/**\n * Parse a `robots.txt`, keeping the rules Googlebot would follow: the\n * `googlebot` group when one exists, the `*` group otherwise.\n */\nexport const parseRobots = (content: string): ParsedRobots => {\n  const groups: RobotsGroup[] = [];\n  const sitemapUrls: string[] = [];\n  let currentGroup: RobotsGroup | undefined;\n  let isReadingUserAgents = false;\n\n  for (const rawLine of content.split(/\\r?\\n/)) {\n    const line = rawLine.replace(/#.*$/, '').trim();\n    const separatorIndex = line.indexOf(':');\n    if (separatorIndex === -1) continue;\n\n    const directive = line.slice(0, separatorIndex).trim().toLowerCase();\n    const value = line.slice(separatorIndex + 1).trim();\n\n    if (directive === 'sitemap') {\n      if (value) sitemapUrls.push(value);\n      continue;\n    }\n\n    if (directive === 'user-agent') {\n      if (!isReadingUserAgents || !currentGroup) {\n        currentGroup = {\n          userAgents: [],\n          disallowedPaths: [],\n          allowedPaths: [],\n        };\n        groups.push(currentGroup);\n      }\n      currentGroup.userAgents.push(value.toLowerCase());\n      isReadingUserAgents = true;\n      continue;\n    }\n\n    isReadingUserAgents = false;\n    if (!currentGroup || !value) continue;\n    if (directive === 'disallow') currentGroup.disallowedPaths.push(value);\n    if (directive === 'allow') currentGroup.allowedPaths.push(value);\n  }\n\n  const googlebotGroups = groups.filter(({ userAgents }) =>\n    userAgents.includes('googlebot')\n  );\n  const applicableGroups =\n    googlebotGroups.length > 0\n      ? googlebotGroups\n      : groups.filter(({ userAgents }) => userAgents.includes('*'));\n\n  return {\n    disallowedPaths: applicableGroups.flatMap(\n      ({ disallowedPaths }) => disallowedPaths\n    ),\n    allowedPaths: applicableGroups.flatMap(({ allowedPaths }) => allowedPaths),\n    sitemapUrls,\n  };\n};\n\n/** Convert a robots.txt path pattern (`*`, `$`) into a RegExp. */\nconst robotsPatternToRegExp = (pattern: string): RegExp =>\n  new RegExp(\n    `^${pattern\n      .replace(/[.+?^${}()|[\\]\\\\]/g, '\\\\$&')\n      .replace(/\\*/g, '.*')\n      .replace(/\\\\\\$$/, '$')}`\n  );\n\n/**\n * Whether Googlebot may crawl `pathWithSearch` (path + query string) under the\n * parsed rules. Follows Google's precedence: the longest matching rule wins,\n * `Allow` wins ties.\n */\nexport const isPathAllowedByRobots = (\n  pathWithSearch: string,\n  robots: Pick<ParsedRobots, 'disallowedPaths' | 'allowedPaths'>\n): boolean => {\n  const longestMatch = (patterns: string[]): number =>\n    patterns.reduce(\n      (longest, pattern) =>\n        robotsPatternToRegExp(pattern).test(pathWithSearch)\n          ? Math.max(longest, pattern.length)\n          : longest,\n      -1\n    );\n\n  const disallowLength = longestMatch(robots.disallowedPaths);\n  if (disallowLength === -1) return true;\n  return longestMatch(robots.allowedPaths) >= disallowLength;\n};\n"],"mappings":";;;;;AAoBA,MAAa,eAAe,YAAkC;CAC5D,MAAM,SAAwB,CAAC;CAC/B,MAAM,cAAwB,CAAC;CAC/B,IAAI;CACJ,IAAI,sBAAsB;CAE1B,KAAK,MAAM,WAAW,QAAQ,MAAM,OAAO,GAAG;EAC5C,MAAM,OAAO,QAAQ,QAAQ,QAAQ,EAAE,CAAC,CAAC,KAAK;EAC9C,MAAM,iBAAiB,KAAK,QAAQ,GAAG;EACvC,IAAI,mBAAmB,IAAI;EAE3B,MAAM,YAAY,KAAK,MAAM,GAAG,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,YAAY;EACnE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,CAAC,CAAC,CAAC,KAAK;EAElD,IAAI,cAAc,WAAW;GAC3B,IAAI,OAAO,YAAY,KAAK,KAAK;GACjC;EACF;EAEA,IAAI,cAAc,cAAc;GAC9B,IAAI,CAAC,uBAAuB,CAAC,cAAc;IACzC,eAAe;KACb,YAAY,CAAC;KACb,iBAAiB,CAAC;KAClB,cAAc,CAAC;IACjB;IACA,OAAO,KAAK,YAAY;GAC1B;GACA,aAAa,WAAW,KAAK,MAAM,YAAY,CAAC;GAChD,sBAAsB;GACtB;EACF;EAEA,sBAAsB;EACtB,IAAI,CAAC,gBAAgB,CAAC,OAAO;EAC7B,IAAI,cAAc,YAAY,aAAa,gBAAgB,KAAK,KAAK;EACrE,IAAI,cAAc,SAAS,aAAa,aAAa,KAAK,KAAK;CACjE;CAEA,MAAM,kBAAkB,OAAO,QAAQ,EAAE,iBACvC,WAAW,SAAS,WAAW,CACjC;CACA,MAAM,mBACJ,gBAAgB,SAAS,IACrB,kBACA,OAAO,QAAQ,EAAE,iBAAiB,WAAW,SAAS,GAAG,CAAC;CAEhE,OAAO;EACL,iBAAiB,iBAAiB,SAC/B,EAAE,sBAAsB,eAC3B;EACA,cAAc,iBAAiB,SAAS,EAAE,mBAAmB,YAAY;EACzE;CACF;AACF;;AAGA,MAAM,yBAAyB,YAC7B,IAAI,OACF,IAAI,QACD,QAAQ,sBAAsB,MAAM,CAAC,CACrC,QAAQ,OAAO,IAAI,CAAC,CACpB,QAAQ,SAAS,GAAG,GACzB;;;;;;AAOF,MAAa,yBACX,gBACA,WACY;CACZ,MAAM,gBAAgB,aACpB,SAAS,QACN,SAAS,YACR,sBAAsB,OAAO,CAAC,CAAC,KAAK,cAAc,IAC9C,KAAK,IAAI,SAAS,QAAQ,MAAM,IAChC,SACN,EACF;CAEF,MAAM,iBAAiB,aAAa,OAAO,eAAe;CAC1D,IAAI,mBAAmB,IAAI,OAAO;CAClC,OAAO,aAAa,OAAO,YAAY,KAAK;AAC9C"}

package/dist/esm/scan/runScanChecks.mjs

@@ -0,0 +1,88 @@
+import { parseUrl } from "./detection/url.mjs";
+import { detectTechnologies } from "./detection/detectTechnologies.mjs";
+import { extractAnchors, extractCanonicalHref, extractHreflangs, extractHtmlDir, extractHtmlLang, extractMetaDescription, extractOgImage, extractOgLocale, extractOgLocaleAlternates, extractResourceUrls, extractTitle } from "./parseHtml.mjs";
+import { detectRoutingStrategy } from "./detection/detectRoutingStrategy.mjs";
+import { checkBundleContent, checkCanonical, checkHreflang, checkHreflangReciprocity, checkHtmlDir, checkHtmlLang, checkInternalLinks, checkLocaleConsistency, checkOgLocale, checkRobots, checkSitemap, checkXDefault } from "./checks.mjs";
+
+//#region src/scan/runScanChecks.ts
+/** Extract title, description and absolute preview image. */
+const extractPageMetadata = (html, targetUrl) => {
+	const ogImage = extractOgImage(html);
+	return {
+		title: extractTitle(html),
+		description: extractMetaDescription(html),
+		image: ogImage ? parseUrl(ogImage, targetUrl)?.href ?? ogImage : ""
+	};
+};
+/**
+* Run every i18n/SEO check on an already loaded page. Single implementation
+* shared by the `intlayer scan` CLI and the hosted audit (`/api/scan`), which
+* only differ in how they load the page.
+*
+* @param input - The loaded page (HTML, chunks, size, runtime signals).
+* @param emit - Receives every check result as soon as it is produced.
+* @param options - Network options and progress callbacks.
+*/
+const runScanChecks = async ({ targetUrl, html, chunks, totalPageSize, requestUrls = [], runtimeSignals }, emit, { onProgress, onPageInfo, ...fetchOptions }) => {
+	const origin = new URL(targetUrl).origin;
+	onProgress?.(20, "Analyzing html attributes and hreflang tags...");
+	const langTag = extractHtmlLang(html);
+	const hreflangs = extractHreflangs(html);
+	const routing = detectRoutingStrategy({
+		pageUrl: targetUrl,
+		htmlLang: langTag,
+		hreflangs
+	});
+	const signals = {
+		targetUrl,
+		langTag,
+		dirTag: extractHtmlDir(html),
+		ogLocale: extractOgLocale(html),
+		ogLocaleAlternates: extractOgLocaleAlternates(html),
+		canonicalHref: extractCanonicalHref(html),
+		hreflangs,
+		anchors: extractAnchors(html),
+		routing,
+		pageLocale: routing.urlLocale ?? langTag
+	};
+	const locales = [.../* @__PURE__ */ new Set([...langTag ? [langTag] : [], ...routing.locales])];
+	const pageInfo = {
+		metadata: extractPageMetadata(html, targetUrl),
+		routing,
+		locales,
+		technologies: detectTechnologies({
+			pageUrl: targetUrl,
+			html,
+			resourceUrls: [.../* @__PURE__ */ new Set([...extractResourceUrls(html, targetUrl), ...requestUrls])],
+			scripts: chunks.map(({ content }) => content),
+			...runtimeSignals
+		})
+	};
+	onPageInfo?.(pageInfo);
+	checkHtmlLang(signals, emit);
+	checkHtmlDir(signals, emit);
+	checkLocaleConsistency(signals, emit);
+	checkOgLocale(signals, emit);
+	checkCanonical(signals, emit);
+	checkHreflang(signals, emit);
+	checkXDefault(signals, emit);
+	onProgress?.(35, "Checking hreflang alternates...");
+	await checkHreflangReciprocity(signals, fetchOptions, emit);
+	onProgress?.(50, "Analysing bundle content & leakage...");
+	const bundle = checkBundleContent(chunks, html, signals.pageLocale, targetUrl, totalPageSize, emit);
+	onProgress?.(60, "Analyzing internal links...");
+	checkInternalLinks(signals, emit);
+	onProgress?.(70, "Checking robots.txt...");
+	const localizedUrls = hreflangs.flatMap(({ href }) => parseUrl(href, targetUrl)?.href ?? []);
+	const declaredSitemapUrls = await checkRobots(origin, localizedUrls, fetchOptions, emit);
+	onProgress?.(85, "Checking sitemaps...");
+	await checkSitemap(origin, declaredSitemapUrls, routing, fetchOptions, emit);
+	return {
+		...pageInfo,
+		bundle
+	};
+};
+
+//#endregion
+export { runScanChecks };
+//# sourceMappingURL=runScanChecks.mjs.map

package/dist/esm/scan/runScanChecks.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"runScanChecks.mjs","names":[],"sources":["../../../src/scan/runScanChecks.ts"],"sourcesContent":["import {\n  checkBundleContent,\n  checkCanonical,\n  checkHreflang,\n  checkHreflangReciprocity,\n  checkHtmlDir,\n  checkHtmlLang,\n  checkInternalLinks,\n  checkLocaleConsistency,\n  checkOgLocale,\n  checkRobots,\n  checkSitemap,\n  checkXDefault,\n  type EmitScanEvent,\n  type PageSignals,\n} from './checks';\nimport { detectRoutingStrategy } from './detection/detectRoutingStrategy';\nimport {\n  detectTechnologies,\n  type TechnologyDetectionInput,\n} from './detection/detectTechnologies';\nimport { parseUrl } from './detection/url';\nimport type { ScanFetchOptions } from './fetchText';\nimport {\n  extractAnchors,\n  extractCanonicalHref,\n  extractHreflangs,\n  extractHtmlDir,\n  extractHtmlLang,\n  extractMetaDescription,\n  extractOgImage,\n  extractOgLocale,\n  extractOgLocaleAlternates,\n  extractResourceUrls,\n  extractTitle,\n} from './parseHtml';\nimport type { BundleChunkInput, PageMetadata, ScanResult } from './types';\n\n/** Page material gathered by the caller (fetch, puppeteer, browser…). */\nexport type ScanPageInput = {\n  /** Absolute URL of the scanned page. */\n  targetUrl: string;\n  /** HTML of the page, rendered when possible. */\n  html: string;\n  /** Fetched JavaScript chunks. */\n  chunks: BundleChunkInput[];\n  /** Total transferred bytes measured for the page. */\n  totalPageSize: number;\n  /** Network request URLs observed while loading the page. */\n  requestUrls?: string[];\n  /** Live-page signals (globals, storage keys) for technology detection. */\n  runtimeSignals?: Pick<\n    TechnologyDetectionInput,\n    'globals' | 'storageKeys' | 'globalVersions'\n  >;\n};\n\n/** Page-level information available before the network-bound checks. */\nexport type ScanPageInfo = Pick<\n  ScanResult,\n  'metadata' | 'routing' | 'technologies' | 'locales'\n>;\n\n/** Options of {@link runScanChecks}. */\nexport type RunScanChecksOptions = ScanFetchOptions & {\n  /** Called before each step, e.g. to stream a progress bar. */\n  onProgress?: (progress: number, message: string) => void;\n  /** Called once the page itself is analyzed (metadata, routing, stack). */\n  onPageInfo?: (pageInfo: ScanPageInfo) => void;\n};\n\n/** Output of {@link runScanChecks}. */\nexport type ScanChecksResult = Pick<\n  ScanResult,\n  'metadata' | 'routing' | 'technologies' | 'locales' | 'bundle'\n>;\n\n/** Extract title, description and absolute preview image. */\nconst extractPageMetadata = (html: string, targetUrl: string): PageMetadata => {\n  const ogImage = extractOgImage(html);\n  return {\n    title: extractTitle(html),\n    description: extractMetaDescription(html),\n    image: ogImage ? (parseUrl(ogImage, targetUrl)?.href ?? ogImage) : '',\n  };\n};\n\n/**\n * Run every i18n/SEO check on an already loaded page. Single implementation\n * shared by the `intlayer scan` CLI and the hosted audit (`/api/scan`), which\n * only differ in how they load the page.\n *\n * @param input - The loaded page (HTML, chunks, size, runtime signals).\n * @param emit - Receives every check result as soon as it is produced.\n * @param options - Network options and progress callbacks.\n */\nexport const runScanChecks = async (\n  {\n    targetUrl,\n    html,\n    chunks,\n    totalPageSize,\n    requestUrls = [],\n    runtimeSignals,\n  }: ScanPageInput,\n  emit: EmitScanEvent,\n  { onProgress, onPageInfo, ...fetchOptions }: RunScanChecksOptions\n): Promise<ScanChecksResult> => {\n  const origin = new URL(targetUrl).origin;\n\n  onProgress?.(20, 'Analyzing html attributes and hreflang tags...');\n\n  const langTag = extractHtmlLang(html);\n  const hreflangs = extractHreflangs(html);\n  const routing = detectRoutingStrategy({\n    pageUrl: targetUrl,\n    htmlLang: langTag,\n    hreflangs,\n  });\n  const signals: PageSignals = {\n    targetUrl,\n    langTag,\n    dirTag: extractHtmlDir(html),\n    ogLocale: extractOgLocale(html),\n    ogLocaleAlternates: extractOgLocaleAlternates(html),\n    canonicalHref: extractCanonicalHref(html),\n    hreflangs,\n    anchors: extractAnchors(html),\n    routing,\n    pageLocale: routing.urlLocale ?? langTag,\n  };\n\n  const locales = [\n    ...new Set([...(langTag ? [langTag] : []), ...routing.locales]),\n  ];\n  const pageInfo: ScanPageInfo = {\n    metadata: extractPageMetadata(html, targetUrl),\n    routing,\n    locales,\n    technologies: detectTechnologies({\n      pageUrl: targetUrl,\n      html,\n      resourceUrls: [\n        ...new Set([...extractResourceUrls(html, targetUrl), ...requestUrls]),\n      ],\n      scripts: chunks.map(({ content }) => content),\n      ...runtimeSignals,\n    }),\n  };\n  onPageInfo?.(pageInfo);\n\n  checkHtmlLang(signals, emit);\n  checkHtmlDir(signals, emit);\n  checkLocaleConsistency(signals, emit);\n  checkOgLocale(signals, emit);\n  checkCanonical(signals, emit);\n  checkHreflang(signals, emit);\n  checkXDefault(signals, emit);\n\n  onProgress?.(35, 'Checking hreflang alternates...');\n  await checkHreflangReciprocity(signals, fetchOptions, emit);\n\n  onProgress?.(50, 'Analysing bundle content & leakage...');\n  const bundle = checkBundleContent(\n    chunks,\n    html,\n    signals.pageLocale,\n    targetUrl,\n    totalPageSize,\n    emit\n  );\n\n  onProgress?.(60, 'Analyzing internal links...');\n  checkInternalLinks(signals, emit);\n\n  onProgress?.(70, 'Checking robots.txt...');\n  const localizedUrls = hreflangs.flatMap(\n    ({ href }) => parseUrl(href, targetUrl)?.href ?? []\n  );\n  const declaredSitemapUrls = await checkRobots(\n    origin,\n    localizedUrls,\n    fetchOptions,\n    emit\n  );\n\n  onProgress?.(85, 'Checking sitemaps...');\n  await checkSitemap(origin, declaredSitemapUrls, routing, fetchOptions, emit);\n\n  return { ...pageInfo, bundle };\n};\n"],"mappings":";;;;;;;;AA8EA,MAAM,uBAAuB,MAAc,cAAoC;CAC7E,MAAM,UAAU,eAAe,IAAI;CACnC,OAAO;EACL,OAAO,aAAa,IAAI;EACxB,aAAa,uBAAuB,IAAI;EACxC,OAAO,UAAW,SAAS,SAAS,SAAS,CAAC,EAAE,QAAQ,UAAW;CACrE;AACF;;;;;;;;;;AAWA,MAAa,gBAAgB,OAC3B,EACE,WACA,MACA,QACA,eACA,cAAc,CAAC,GACf,kBAEF,MACA,EAAE,YAAY,YAAY,GAAG,mBACC;CAC9B,MAAM,SAAS,IAAI,IAAI,SAAS,CAAC,CAAC;CAElC,aAAa,IAAI,gDAAgD;CAEjE,MAAM,UAAU,gBAAgB,IAAI;CACpC,MAAM,YAAY,iBAAiB,IAAI;CACvC,MAAM,UAAU,sBAAsB;EACpC,SAAS;EACT,UAAU;EACV;CACF,CAAC;CACD,MAAM,UAAuB;EAC3B;EACA;EACA,QAAQ,eAAe,IAAI;EAC3B,UAAU,gBAAgB,IAAI;EAC9B,oBAAoB,0BAA0B,IAAI;EAClD,eAAe,qBAAqB,IAAI;EACxC;EACA,SAAS,eAAe,IAAI;EAC5B;EACA,YAAY,QAAQ,aAAa;CACnC;CAEA,MAAM,UAAU,CACd,mBAAG,IAAI,IAAI,CAAC,GAAI,UAAU,CAAC,OAAO,IAAI,CAAC,GAAI,GAAG,QAAQ,OAAO,CAAC,CAChE;CACA,MAAM,WAAyB;EAC7B,UAAU,oBAAoB,MAAM,SAAS;EAC7C;EACA;EACA,cAAc,mBAAmB;GAC/B,SAAS;GACT;GACA,cAAc,CACZ,mBAAG,IAAI,IAAI,CAAC,GAAG,oBAAoB,MAAM,SAAS,GAAG,GAAG,WAAW,CAAC,CACtE;GACA,SAAS,OAAO,KAAK,EAAE,cAAc,OAAO;GAC5C,GAAG;EACL,CAAC;CACH;CACA,aAAa,QAAQ;CAErB,cAAc,SAAS,IAAI;CAC3B,aAAa,SAAS,IAAI;CAC1B,uBAAuB,SAAS,IAAI;CACpC,cAAc,SAAS,IAAI;CAC3B,eAAe,SAAS,IAAI;CAC5B,cAAc,SAAS,IAAI;CAC3B,cAAc,SAAS,IAAI;CAE3B,aAAa,IAAI,iCAAiC;CAClD,MAAM,yBAAyB,SAAS,cAAc,IAAI;CAE1D,aAAa,IAAI,uCAAuC;CACxD,MAAM,SAAS,mBACb,QACA,MACA,QAAQ,YACR,WACA,eACA,IACF;CAEA,aAAa,IAAI,6BAA6B;CAC9C,mBAAmB,SAAS,IAAI;CAEhC,aAAa,IAAI,wBAAwB;CACzC,MAAM,gBAAgB,UAAU,SAC7B,EAAE,WAAW,SAAS,MAAM,SAAS,CAAC,EAAE,QAAQ,CAAC,CACpD;CACA,MAAM,sBAAsB,MAAM,YAChC,QACA,eACA,cACA,IACF;CAEA,aAAa,IAAI,sBAAsB;CACvC,MAAM,aAAa,QAAQ,qBAAqB,SAAS,cAAc,IAAI;CAE3E,OAAO;EAAE,GAAG;EAAU;CAAO;AAC/B"}

package/dist/esm/scan/scanWebsite.mjs

@@ -1,6 +1,8 @@
+import { fetchText } from "./fetchText.mjs";
 import { mutateScore, toScorePercent } from "./calculateScore.mjs";
-import { byteLength, extractAnchors, extractScriptUrls } from "./parseHtml.mjs";
-import { checkBundleContent, checkCanonical, checkHtmlAttributes, checkLinguisticStructure, checkRobots, checkSitemap, checkUrlStructure } from "./checks.mjs";
+import { getTechnologyGlobalNames } from "./detection/detectTechnologies.mjs";
+import { byteLength, extractScriptUrls } from "./parseHtml.mjs";
+import { runScanChecks } from "./runScanChecks.mjs";
 import { GREY, GREY_LIGHT } from "@intlayer/config/colors";
 import { colorize, logger } from "@intlayer/config/logger";
 
@@ -20,6 +22,22 @@ const logDeepScanRecommendation = () => {
 	]);
 };
 /**
+* Read, in the page context, which technology globals exist and the version
+* paths they expose. Serialized by puppeteer: must stay self-contained.
+*/
+const readRuntimeGlobals = (globalNames, versionPaths) => {
+	const pageWindow = window;
+	const globalVersions = {};
+	for (const versionPath of versionPaths) {
+		const value = versionPath.split(".").reduce((current, key) => current && typeof current === "object" ? current[key] : void 0, pageWindow);
+		if (typeof value === "string") globalVersions[versionPath] = value;
+	}
+	return {
+		globals: globalNames.filter((name) => pageWindow[name] !== void 0),
+		globalVersions
+	};
+};
+/**
 * Render the page with a locally installed `puppeteer` to capture
 * client-rendered content, the accurate transfer size, and lazy-loaded chunks.
 *
@@ -55,11 +73,13 @@ const runDeepScan = async (targetUrl, userAgent, timeoutMs) => {
 		await page.setExtraHTTPHeaders({ "Accept-Language": "en-US,en;q=0.9" });
 		const origin = new URL(targetUrl).origin;
 		const jsResponseMap = /* @__PURE__ */ new Map();
+		const requestUrls = [];
 		let totalPageSize = 0;
 		const pendingResponses = [];
 		page.on("response", (response) => {
 			pendingResponses.push((async () => {
 				try {
+					requestUrls.push(response.url());
 					if (response.status() !== 200) return;
 					const buffer = await response.buffer();
 					totalPageSize += buffer.length;
@@ -84,35 +104,27 @@ const runDeepScan = async (targetUrl, userAgent, timeoutMs) => {
 			isMainBundle: mainBundleUrls.has(url),
 			content
 		}));
+		const { globals: globalNames, versionGlobals } = getTechnologyGlobalNames();
+		const { globals, globalVersions } = await page.evaluate(readRuntimeGlobals, globalNames, versionGlobals).catch(() => ({
+			globals: [],
+			globalVersions: {}
+		}));
+		const cookies = await page.cookies().catch(() => []);
 		return {
 			html,
 			totalPageSize,
-			chunks
+			chunks,
+			requestUrls,
+			runtimeSignals: {
+				globals,
+				globalVersions,
+				storageKeys: cookies.map(({ name }) => name)
+			}
 		};
 	} finally {
 		if (browser) await browser.close();
 	}
 };
-/** Fetch the raw HTML document, measuring its byte size. */
-const fetchHtml = async (url, userAgent, timeoutMs) => {
-	const controller = new AbortController();
-	const timer = setTimeout(() => controller.abort(), timeoutMs);
-	try {
-		const response = await fetch(url, {
-			headers: {
-				"User-Agent": userAgent,
-				"Accept-Language": "en-US,en;q=0.9"
-			},
-			signal: controller.signal
-		});
-		return {
-			html: await response.text(),
-			finalUrl: response.url || url
-		};
-	} finally {
-		clearTimeout(timer);
-	}
-};
 /**
 * Fetch every eagerly-loaded script. Same-origin scripts keep their content so
 * their locale weight can be analyzed; third-party scripts only contribute to
@@ -153,49 +165,43 @@ const fetchScripts = async (scriptUrls, origin, userAgent) => {
 const scanWebsite = async (targetUrl, options = {}) => {
 	const { deep = true, timeoutMs = DEFAULT_TIMEOUT_MS, userAgent = DEFAULT_USER_AGENT } = options;
 	const origin = new URL(targetUrl).origin;
-	let mode = "basic";
-	let html;
-	let totalPageSize;
-	let chunks;
+	const fetchOptions = {
+		userAgent,
+		timeoutMs
+	};
 	const deepResult = deep ? await runDeepScan(targetUrl, userAgent, timeoutMs) : null;
-	if (deepResult) {
-		mode = "deep";
-		html = deepResult.html;
-		chunks = deepResult.chunks;
-		totalPageSize = deepResult.totalPageSize;
-	} else {
+	let pageInput;
+	if (deepResult) pageInput = deepResult;
+	else {
 		if (deep) logDeepScanRecommendation();
-		const { html: fetchedHtml, finalUrl } = await fetchHtml(targetUrl, userAgent, timeoutMs);
-		html = fetchedHtml;
-		const scriptUrls = extractScriptUrls(fetchedHtml, finalUrl);
-		const { chunks: fetchedChunks, scriptBytes } = await fetchScripts(scriptUrls, origin, userAgent);
-		chunks = fetchedChunks;
-		totalPageSize = byteLength(fetchedHtml) + scriptBytes;
+		const response = await fetchText(targetUrl, fetchOptions);
+		if (!response?.ok) throw new Error(`Failed to fetch ${targetUrl}${response ? ` (HTTP ${response.status})` : ""}`);
+		const scriptUrls = extractScriptUrls(response.text, response.finalUrl);
+		const { chunks, scriptBytes } = await fetchScripts(scriptUrls, origin, userAgent);
+		pageInput = {
+			html: response.text,
+			chunks,
+			totalPageSize: byteLength(response.text) + scriptBytes
+		};
 	}
-	const htmlSize = byteLength(html);
 	const events = [];
-	const localesSet = /* @__PURE__ */ new Set();
-	const { langTag } = checkHtmlAttributes(html, targetUrl, events);
-	checkCanonical(html, targetUrl, events);
-	checkLinguisticStructure(html, targetUrl, localesSet, events);
-	checkUrlStructure(extractAnchors(html), origin, targetUrl, events);
-	const bundle = checkBundleContent(chunks, html, langTag, targetUrl, totalPageSize, events);
-	await checkRobots(origin, localesSet, userAgent, events);
-	await checkSitemap(origin, localesSet, userAgent, events);
+	const checksResult = await runScanChecks({
+		targetUrl,
+		...pageInput
+	}, (event) => events.push(event), fetchOptions);
 	const rawScore = events.reduce((score, event) => mutateScore(score, event), {
 		score: 0,
 		totalScore: 0
 	});
 	return {
 		url: targetUrl,
-		mode,
-		totalPageSize,
-		htmlSize,
+		mode: deepResult ? "deep" : "basic",
+		totalPageSize: pageInput.totalPageSize,
+		htmlSize: byteLength(pageInput.html),
 		score: toScorePercent(rawScore),
 		rawScore,
 		events,
-		locales: Array.from(localesSet),
-		bundle
+		...checksResult
 	};
 };