@intlayer/engine 9.5.10 → 9.5.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/installSkills/skills/bundle-optimization.md +123 -0
- package/dist/assets/installSkills/skills/compat.md +0 -4
- package/dist/assets/installSkills/skills/content.md +8 -2
- package/dist/assets/installSkills/skills/next-js.md +2 -2
- package/dist/assets/installSkills/skills/usage.md +1 -1
- package/dist/cjs/build.cjs +5 -0
- package/dist/cjs/cli.cjs +1 -0
- package/dist/cjs/docReview/alignBlocks.cjs +53 -36
- package/dist/cjs/docReview/alignBlocks.cjs.map +1 -1
- package/dist/cjs/docReview/rebuildDocument.cjs +3 -1
- package/dist/cjs/docReview/rebuildDocument.cjs.map +1 -1
- package/dist/cjs/docReview/segmentDocument.cjs +8 -5
- package/dist/cjs/docReview/segmentDocument.cjs.map +1 -1
- package/dist/cjs/init/frameworkSetup/nextAppRouter/transforms.cjs +4 -1
- package/dist/cjs/init/frameworkSetup/nextAppRouter/transforms.cjs.map +1 -1
- package/dist/cjs/init/index.cjs +2 -12
- package/dist/cjs/init/index.cjs.map +1 -1
- package/dist/cjs/init/utils/backendPackages.cjs +15 -0
- package/dist/cjs/init/utils/backendPackages.cjs.map +1 -0
- package/dist/cjs/init/utils/index.cjs +2 -3
- package/dist/cjs/installMCP/installMCP.cjs +1 -1
- package/dist/cjs/installMCP/installMCP.cjs.map +1 -1
- package/dist/cjs/installSkills/index.cjs +2 -0
- package/dist/cjs/installSkills/index.cjs.map +1 -1
- package/dist/cjs/loadDictionaries/loadDictionaries.cjs +9 -8
- package/dist/cjs/loadDictionaries/loadDictionaries.cjs.map +1 -1
- package/dist/cjs/loadDictionaries/loadLocalDictionaries.cjs +3 -1
- package/dist/cjs/loadDictionaries/loadLocalDictionaries.cjs.map +1 -1
- package/dist/cjs/loadDictionaries/loadRemoteDictionaries.cjs +4 -6
- package/dist/cjs/loadDictionaries/loadRemoteDictionaries.cjs.map +1 -1
- package/dist/cjs/loadDictionaries/log.cjs +1 -1
- package/dist/cjs/loadDictionaries/log.cjs.map +1 -1
- package/dist/cjs/loadDictionaries/logTypeScriptErrors.cjs +19 -12
- package/dist/cjs/loadDictionaries/logTypeScriptErrors.cjs.map +1 -1
- package/dist/cjs/loadDictionaries/remoteDictionaryClient.cjs +18 -0
- package/dist/cjs/loadDictionaries/remoteDictionaryClient.cjs.map +1 -0
- package/dist/cjs/prepareIntlayerServer.cjs +132 -0
- package/dist/cjs/prepareIntlayerServer.cjs.map +1 -0
- package/dist/cjs/scan/analyzeBundleContent.cjs +26 -12
- package/dist/cjs/scan/analyzeBundleContent.cjs.map +1 -1
- package/dist/cjs/scan/calculateScore.cjs +4 -1
- package/dist/cjs/scan/calculateScore.cjs.map +1 -1
- package/dist/cjs/scan/checks.cjs +394 -221
- package/dist/cjs/scan/checks.cjs.map +1 -1
- package/dist/cjs/scan/detection/checkDetails.cjs +56 -0
- package/dist/cjs/scan/detection/checkDetails.cjs.map +1 -0
- package/dist/cjs/scan/detection/classifyInternalLinks.cjs +47 -0
- package/dist/cjs/scan/detection/classifyInternalLinks.cjs.map +1 -0
- package/dist/cjs/scan/detection/detectRoutingStrategy.cjs +143 -0
- package/dist/cjs/scan/detection/detectRoutingStrategy.cjs.map +1 -0
- package/dist/cjs/scan/detection/detectTechnologies.cjs +89 -0
- package/dist/cjs/scan/detection/detectTechnologies.cjs.map +1 -0
- package/dist/cjs/scan/detection/index.cjs +36 -0
- package/dist/cjs/scan/detection/localeCode.cjs +118 -0
- package/dist/cjs/scan/detection/localeCode.cjs.map +1 -0
- package/dist/cjs/scan/detection/localizedPages.cjs +48 -0
- package/dist/cjs/scan/detection/localizedPages.cjs.map +1 -0
- package/dist/cjs/scan/detection/technologySignatures.cjs +564 -0
- package/dist/cjs/scan/detection/technologySignatures.cjs.map +1 -0
- package/dist/cjs/scan/detection/url.cjs +53 -0
- package/dist/cjs/scan/detection/url.cjs.map +1 -0
- package/dist/cjs/scan/fetchText.cjs +56 -0
- package/dist/cjs/scan/fetchText.cjs.map +1 -0
- package/dist/cjs/scan/index.cjs +59 -3
- package/dist/cjs/scan/parseHtml.cjs +55 -10
- package/dist/cjs/scan/parseHtml.cjs.map +1 -1
- package/dist/cjs/scan/robots.cjs +65 -0
- package/dist/cjs/scan/robots.cjs.map +1 -0
- package/dist/cjs/scan/runScanChecks.cjs +89 -0
- package/dist/cjs/scan/runScanChecks.cjs.map +1 -0
- package/dist/cjs/scan/scanWebsite.cjs +58 -52
- package/dist/cjs/scan/scanWebsite.cjs.map +1 -1
- package/dist/cjs/scan/sitemap.cjs +109 -0
- package/dist/cjs/scan/sitemap.cjs.map +1 -0
- package/dist/cjs/utils/chunkJSON.cjs +24 -9
- package/dist/cjs/utils/chunkJSON.cjs.map +1 -1
- package/dist/cjs/utils/getChunk.cjs +2 -2
- package/dist/cjs/utils/getChunk.cjs.map +1 -1
- package/dist/cjs/utils/getProcessChainCommands.cjs +117 -0
- package/dist/cjs/utils/getProcessChainCommands.cjs.map +1 -0
- package/dist/cjs/utils/index.cjs +2 -0
- package/dist/cjs/utils/runParallel/index.cjs +1 -1
- package/dist/cjs/utils/runParallel/index.cjs.map +1 -1
- package/dist/cjs/utils/runParallel/pidTree.cjs +23 -11
- package/dist/cjs/utils/runParallel/pidTree.cjs.map +1 -1
- package/dist/cjs/utils/runParallel/ps.cjs +8 -3
- package/dist/cjs/utils/runParallel/ps.cjs.map +1 -1
- package/dist/cjs/utils/runParallel/runTask.cjs +2 -1
- package/dist/cjs/utils/runParallel/runTask.cjs.map +1 -1
- package/dist/cjs/utils/runParallel/wmic.cjs +8 -3
- package/dist/cjs/utils/runParallel/wmic.cjs.map +1 -1
- package/dist/cjs/utils/startContentWatcher.cjs +99 -0
- package/dist/cjs/utils/startContentWatcher.cjs.map +1 -0
- package/dist/cjs/watcher.cjs +2 -2
- package/dist/cjs/watcher.cjs.map +1 -1
- package/dist/cjs/writeContentDeclaration/detectExportedComponentName.cjs +7 -4
- package/dist/cjs/writeContentDeclaration/detectExportedComponentName.cjs.map +1 -1
- package/dist/cjs/writeContentDeclaration/writeMarkdownFile.cjs +2 -2
- package/dist/cjs/writeContentDeclaration/writeMarkdownFile.cjs.map +1 -1
- package/dist/cjs/writeJsonIfChanged.cjs.map +1 -1
- package/dist/esm/build.mjs +2 -1
- package/dist/esm/cli.mjs +2 -2
- package/dist/esm/docReview/alignBlocks.mjs +53 -36
- package/dist/esm/docReview/alignBlocks.mjs.map +1 -1
- package/dist/esm/docReview/rebuildDocument.mjs +3 -1
- package/dist/esm/docReview/rebuildDocument.mjs.map +1 -1
- package/dist/esm/docReview/segmentDocument.mjs +8 -5
- package/dist/esm/docReview/segmentDocument.mjs.map +1 -1
- package/dist/esm/init/frameworkSetup/nextAppRouter/transforms.mjs +4 -1
- package/dist/esm/init/frameworkSetup/nextAppRouter/transforms.mjs.map +1 -1
- package/dist/esm/init/index.mjs +1 -11
- package/dist/esm/init/index.mjs.map +1 -1
- package/dist/esm/init/utils/backendPackages.mjs +14 -0
- package/dist/esm/init/utils/backendPackages.mjs.map +1 -0
- package/dist/esm/init/utils/index.mjs +2 -2
- package/dist/esm/installMCP/installMCP.mjs +1 -1
- package/dist/esm/installMCP/installMCP.mjs.map +1 -1
- package/dist/esm/installSkills/index.mjs +2 -0
- package/dist/esm/installSkills/index.mjs.map +1 -1
- package/dist/esm/loadDictionaries/loadDictionaries.mjs +9 -8
- package/dist/esm/loadDictionaries/loadDictionaries.mjs.map +1 -1
- package/dist/esm/loadDictionaries/loadLocalDictionaries.mjs +3 -1
- package/dist/esm/loadDictionaries/loadLocalDictionaries.mjs.map +1 -1
- package/dist/esm/loadDictionaries/loadRemoteDictionaries.mjs +3 -5
- package/dist/esm/loadDictionaries/loadRemoteDictionaries.mjs.map +1 -1
- package/dist/esm/loadDictionaries/log.mjs +1 -1
- package/dist/esm/loadDictionaries/log.mjs.map +1 -1
- package/dist/esm/loadDictionaries/logTypeScriptErrors.mjs +19 -12
- package/dist/esm/loadDictionaries/logTypeScriptErrors.mjs.map +1 -1
- package/dist/esm/loadDictionaries/remoteDictionaryClient.mjs +16 -0
- package/dist/esm/loadDictionaries/remoteDictionaryClient.mjs.map +1 -0
- package/dist/esm/prepareIntlayerServer.mjs +128 -0
- package/dist/esm/prepareIntlayerServer.mjs.map +1 -0
- package/dist/esm/scan/analyzeBundleContent.mjs +26 -12
- package/dist/esm/scan/analyzeBundleContent.mjs.map +1 -1
- package/dist/esm/scan/calculateScore.mjs +4 -1
- package/dist/esm/scan/calculateScore.mjs.map +1 -1
- package/dist/esm/scan/checks.mjs +387 -220
- package/dist/esm/scan/checks.mjs.map +1 -1
- package/dist/esm/scan/detection/checkDetails.mjs +54 -0
- package/dist/esm/scan/detection/checkDetails.mjs.map +1 -0
- package/dist/esm/scan/detection/classifyInternalLinks.mjs +46 -0
- package/dist/esm/scan/detection/classifyInternalLinks.mjs.map +1 -0
- package/dist/esm/scan/detection/detectRoutingStrategy.mjs +141 -0
- package/dist/esm/scan/detection/detectRoutingStrategy.mjs.map +1 -0
- package/dist/esm/scan/detection/detectTechnologies.mjs +87 -0
- package/dist/esm/scan/detection/detectTechnologies.mjs.map +1 -0
- package/dist/esm/scan/detection/index.mjs +10 -0
- package/dist/esm/scan/detection/localeCode.mjs +108 -0
- package/dist/esm/scan/detection/localeCode.mjs.map +1 -0
- package/dist/esm/scan/detection/localizedPages.mjs +46 -0
- package/dist/esm/scan/detection/localizedPages.mjs.map +1 -0
- package/dist/esm/scan/detection/technologySignatures.mjs +563 -0
- package/dist/esm/scan/detection/technologySignatures.mjs.map +1 -0
- package/dist/esm/scan/detection/url.mjs +47 -0
- package/dist/esm/scan/detection/url.mjs.map +1 -0
- package/dist/esm/scan/fetchText.mjs +55 -0
- package/dist/esm/scan/fetchText.mjs.map +1 -0
- package/dist/esm/scan/index.mjs +15 -3
- package/dist/esm/scan/parseHtml.mjs +52 -11
- package/dist/esm/scan/parseHtml.mjs.map +1 -1
- package/dist/esm/scan/robots.mjs +63 -0
- package/dist/esm/scan/robots.mjs.map +1 -0
- package/dist/esm/scan/runScanChecks.mjs +88 -0
- package/dist/esm/scan/runScanChecks.mjs.map +1 -0
- package/dist/esm/scan/scanWebsite.mjs +59 -53
- package/dist/esm/scan/scanWebsite.mjs.map +1 -1
- package/dist/esm/scan/sitemap.mjs +105 -0
- package/dist/esm/scan/sitemap.mjs.map +1 -0
- package/dist/esm/utils/chunkJSON.mjs +24 -9
- package/dist/esm/utils/chunkJSON.mjs.map +1 -1
- package/dist/esm/utils/getChunk.mjs +2 -2
- package/dist/esm/utils/getChunk.mjs.map +1 -1
- package/dist/esm/utils/getProcessChainCommands.mjs +115 -0
- package/dist/esm/utils/getProcessChainCommands.mjs.map +1 -0
- package/dist/esm/utils/index.mjs +2 -1
- package/dist/esm/utils/runParallel/index.mjs +1 -1
- package/dist/esm/utils/runParallel/index.mjs.map +1 -1
- package/dist/esm/utils/runParallel/pidTree.mjs +23 -11
- package/dist/esm/utils/runParallel/pidTree.mjs.map +1 -1
- package/dist/esm/utils/runParallel/ps.mjs +8 -3
- package/dist/esm/utils/runParallel/ps.mjs.map +1 -1
- package/dist/esm/utils/runParallel/runTask.mjs +2 -1
- package/dist/esm/utils/runParallel/runTask.mjs.map +1 -1
- package/dist/esm/utils/runParallel/wmic.mjs +8 -3
- package/dist/esm/utils/runParallel/wmic.mjs.map +1 -1
- package/dist/esm/utils/startContentWatcher.mjs +98 -0
- package/dist/esm/utils/startContentWatcher.mjs.map +1 -0
- package/dist/esm/watcher.mjs +1 -1
- package/dist/esm/watcher.mjs.map +1 -1
- package/dist/esm/writeContentDeclaration/detectExportedComponentName.mjs +7 -4
- package/dist/esm/writeContentDeclaration/detectExportedComponentName.mjs.map +1 -1
- package/dist/esm/writeContentDeclaration/writeMarkdownFile.mjs +2 -2
- package/dist/esm/writeContentDeclaration/writeMarkdownFile.mjs.map +1 -1
- package/dist/esm/writeJsonIfChanged.mjs +1 -1
- package/dist/esm/writeJsonIfChanged.mjs.map +1 -1
- package/dist/types/build.d.ts +2 -1
- package/dist/types/cli.d.ts +2 -2
- package/dist/types/docReview/rebuildDocument.d.ts.map +1 -1
- package/dist/types/docReview/segmentDocument.d.ts.map +1 -1
- package/dist/types/init/frameworkSetup/nextAppRouter/transforms.d.ts.map +1 -1
- package/dist/types/init/index.d.ts +1 -1
- package/dist/types/init/index.d.ts.map +1 -1
- package/dist/types/init/utils/backendPackages.d.ts +5 -0
- package/dist/types/init/utils/backendPackages.d.ts.map +1 -0
- package/dist/types/init/utils/index.d.ts +2 -2
- package/dist/types/installSkills/index.d.ts +1 -0
- package/dist/types/installSkills/index.d.ts.map +1 -1
- package/dist/types/loadDictionaries/loadRemoteDictionaries.d.ts.map +1 -1
- package/dist/types/loadDictionaries/logTypeScriptErrors.d.ts.map +1 -1
- package/dist/types/loadDictionaries/remoteDictionaryClient.d.ts +14 -0
- package/dist/types/loadDictionaries/remoteDictionaryClient.d.ts.map +1 -0
- package/dist/types/prepareIntlayerServer.d.ts +67 -0
- package/dist/types/prepareIntlayerServer.d.ts.map +1 -0
- package/dist/types/scan/analyzeBundleContent.d.ts.map +1 -1
- package/dist/types/scan/calculateScore.d.ts +4 -1
- package/dist/types/scan/calculateScore.d.ts.map +1 -1
- package/dist/types/scan/checks.d.ts +114 -21
- package/dist/types/scan/checks.d.ts.map +1 -1
- package/dist/types/scan/detection/checkDetails.d.ts +29 -0
- package/dist/types/scan/detection/checkDetails.d.ts.map +1 -0
- package/dist/types/scan/detection/classifyInternalLinks.d.ts +44 -0
- package/dist/types/scan/detection/classifyInternalLinks.d.ts.map +1 -0
- package/dist/types/scan/detection/detectRoutingStrategy.d.ts +60 -0
- package/dist/types/scan/detection/detectRoutingStrategy.d.ts.map +1 -0
- package/dist/types/scan/detection/detectTechnologies.d.ts +60 -0
- package/dist/types/scan/detection/detectTechnologies.d.ts.map +1 -0
- package/dist/types/scan/detection/index.d.ts +9 -0
- package/dist/types/scan/detection/localeCode.d.ts +49 -0
- package/dist/types/scan/detection/localeCode.d.ts.map +1 -0
- package/dist/types/scan/detection/localizedPages.d.ts +40 -0
- package/dist/types/scan/detection/localizedPages.d.ts.map +1 -0
- package/dist/types/scan/detection/technologySignatures.d.ts +48 -0
- package/dist/types/scan/detection/technologySignatures.d.ts.map +1 -0
- package/dist/types/scan/detection/url.d.ts +27 -0
- package/dist/types/scan/detection/url.d.ts.map +1 -0
- package/dist/types/scan/fetchText.d.ts +30 -0
- package/dist/types/scan/fetchText.d.ts.map +1 -0
- package/dist/types/scan/index.d.ts +16 -4
- package/dist/types/scan/parseHtml.d.ts +17 -0
- package/dist/types/scan/parseHtml.d.ts.map +1 -1
- package/dist/types/scan/robots.d.ts +23 -0
- package/dist/types/scan/robots.d.ts.map +1 -0
- package/dist/types/scan/runScanChecks.d.ts +43 -0
- package/dist/types/scan/runScanChecks.d.ts.map +1 -0
- package/dist/types/scan/scanWebsite.d.ts.map +1 -1
- package/dist/types/scan/sitemap.d.ts +49 -0
- package/dist/types/scan/sitemap.d.ts.map +1 -0
- package/dist/types/scan/types.d.ts +15 -0
- package/dist/types/scan/types.d.ts.map +1 -1
- package/dist/types/utils/chunkJSON.d.ts.map +1 -1
- package/dist/types/utils/getProcessChainCommands.d.ts +35 -0
- package/dist/types/utils/getProcessChainCommands.d.ts.map +1 -0
- package/dist/types/utils/index.d.ts +2 -1
- package/dist/types/utils/runParallel/pidTree.d.ts.map +1 -1
- package/dist/types/utils/startContentWatcher.d.ts +40 -0
- package/dist/types/utils/startContentWatcher.d.ts.map +1 -0
- package/dist/types/watcher.d.ts.map +1 -1
- package/dist/types/writeContentDeclaration/writeMarkdownFile.d.ts.map +1 -1
- package/package.json +15 -7
- package/dist/cjs/init/utils/devScript.cjs +0 -57
- package/dist/cjs/init/utils/devScript.cjs.map +0 -1
- package/dist/esm/init/utils/devScript.mjs +0 -55
- package/dist/esm/init/utils/devScript.mjs.map +0 -1
- package/dist/types/init/utils/devScript.d.ts +0 -48
- package/dist/types/init/utils/devScript.d.ts.map +0 -1
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
|
|
2
|
+
//#region src/scan/fetchText.ts
|
|
3
|
+
const MAX_REDIRECTS = 5;
|
|
4
|
+
/**
|
|
5
|
+
* Read a response body as text, gunzipping it when the payload itself is
|
|
6
|
+
* gzip-compressed (e.g. `sitemap.xml.gz`, served without `Content-Encoding`).
|
|
7
|
+
*/
|
|
8
|
+
const readBodyAsText = async (response) => {
|
|
9
|
+
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
10
|
+
if (!(bytes[0] === 31 && bytes[1] === 139)) return new TextDecoder().decode(bytes);
|
|
11
|
+
const decompressedStream = new Blob([bytes]).stream().pipeThrough(new DecompressionStream("gzip"));
|
|
12
|
+
return new Response(decompressedStream).text();
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* GET a URL as text with a timeout, returning `undefined` when the request is
|
|
16
|
+
* blocked by {@link ScanFetchOptions.shouldFetchUrl} or fails at network level.
|
|
17
|
+
* Redirects are followed manually so every hop goes through the guard.
|
|
18
|
+
*/
|
|
19
|
+
const fetchText = async (url, { userAgent, timeoutMs, shouldFetchUrl }) => {
|
|
20
|
+
const controller = new AbortController();
|
|
21
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
22
|
+
try {
|
|
23
|
+
let currentUrl = url;
|
|
24
|
+
for (let redirectCount = 0;; redirectCount++) {
|
|
25
|
+
if (shouldFetchUrl && !await shouldFetchUrl(currentUrl)) return;
|
|
26
|
+
const response = await fetch(currentUrl, {
|
|
27
|
+
headers: {
|
|
28
|
+
"User-Agent": userAgent,
|
|
29
|
+
"Accept-Language": "en-US,en;q=0.9"
|
|
30
|
+
},
|
|
31
|
+
redirect: "manual",
|
|
32
|
+
signal: controller.signal
|
|
33
|
+
});
|
|
34
|
+
const location = response.headers.get("location");
|
|
35
|
+
if (response.status >= 300 && response.status < 400 && location && redirectCount < MAX_REDIRECTS) {
|
|
36
|
+
currentUrl = new URL(location, currentUrl).href;
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
return {
|
|
40
|
+
status: response.status,
|
|
41
|
+
ok: response.ok,
|
|
42
|
+
text: await readBodyAsText(response),
|
|
43
|
+
finalUrl: currentUrl,
|
|
44
|
+
redirected: redirectCount > 0
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
} catch {
|
|
48
|
+
return;
|
|
49
|
+
} finally {
|
|
50
|
+
clearTimeout(timer);
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
//#endregion
|
|
55
|
+
exports.fetchText = fetchText;
|
|
56
|
+
//# sourceMappingURL=fetchText.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetchText.cjs","names":[],"sources":["../../../src/scan/fetchText.ts"],"sourcesContent":["/** Options shared by every network request issued during a scan. */\nexport type ScanFetchOptions = {\n userAgent: string;\n timeoutMs: number;\n /**\n * Guard called before fetching any URL discovered on the scanned site\n * (hreflang alternates, sitemap children…). The hosted audit uses it to\n * block private / reserved addresses (SSRF). Defaults to allowing all.\n */\n shouldFetchUrl?: (url: string) => boolean | Promise<boolean>;\n};\n\n/** Response of {@link fetchText}. */\nexport type FetchTextResult = {\n status: number;\n ok: boolean;\n text: string;\n /** URL after redirects. */\n finalUrl: string;\n /** Whether at least one redirect was followed. */\n redirected: boolean;\n};\n\nconst MAX_REDIRECTS = 5;\n\n/**\n * Read a response body as text, gunzipping it when the payload itself is\n * gzip-compressed (e.g. `sitemap.xml.gz`, served without `Content-Encoding`).\n */\nconst readBodyAsText = async (response: Response): Promise<string> => {\n const bytes = new Uint8Array(await response.arrayBuffer());\n const isGzip = bytes[0] === 0x1f && bytes[1] === 0x8b;\n if (!isGzip) return new TextDecoder().decode(bytes);\n\n const decompressedStream = new Blob([bytes])\n .stream()\n .pipeThrough(new DecompressionStream('gzip'));\n return new Response(decompressedStream).text();\n};\n\n/**\n * GET a URL as text with a timeout, returning `undefined` when the request is\n * blocked by {@link ScanFetchOptions.shouldFetchUrl} or fails at network level.\n * Redirects are followed manually so every hop goes through the guard.\n */\nexport const fetchText = async (\n url: string,\n { userAgent, timeoutMs, shouldFetchUrl }: ScanFetchOptions\n): Promise<FetchTextResult | undefined> => {\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), timeoutMs);\n try {\n let currentUrl = url;\n for (let redirectCount = 0; ; redirectCount++) {\n if (shouldFetchUrl && !(await shouldFetchUrl(currentUrl))) {\n return undefined;\n }\n const response = await fetch(currentUrl, {\n headers: {\n 'User-Agent': userAgent,\n 'Accept-Language': 'en-US,en;q=0.9',\n },\n redirect: 'manual',\n signal: controller.signal,\n });\n const location = response.headers.get('location');\n if (\n response.status >= 300 &&\n response.status < 400 &&\n location &&\n redirectCount < MAX_REDIRECTS\n ) {\n currentUrl = new URL(location, currentUrl).href;\n continue;\n }\n return {\n status: response.status,\n ok: response.ok,\n text: await readBodyAsText(response),\n finalUrl: currentUrl,\n redirected: redirectCount > 0,\n };\n }\n } catch {\n return undefined;\n } finally {\n clearTimeout(timer);\n }\n};\n"],"mappings":";;AAuBA,MAAM,gBAAgB;;;;;AAMtB,MAAM,iBAAiB,OAAO,aAAwC;CACpE,MAAM,QAAQ,IAAI,WAAW,MAAM,SAAS,YAAY,CAAC;CAEzD,IAAI,EADW,MAAM,OAAO,MAAQ,MAAM,OAAO,MACpC,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK;CAElD,MAAM,qBAAqB,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CACzC,OAAO,CAAC,CACR,YAAY,IAAI,oBAAoB,MAAM,CAAC;CAC9C,OAAO,IAAI,SAAS,kBAAkB,CAAC,CAAC,KAAK;AAC/C;;;;;;AAOA,MAAa,YAAY,OACvB,KACA,EAAE,WAAW,WAAW,qBACiB;CACzC,MAAM,aAAa,IAAI,gBAAgB;CACvC,MAAM,QAAQ,iBAAiB,WAAW,MAAM,GAAG,SAAS;CAC5D,IAAI;EACF,IAAI,aAAa;EACjB,KAAK,IAAI,gBAAgB,IAAK,iBAAiB;GAC7C,IAAI,kBAAkB,CAAE,MAAM,eAAe,UAAU,GACrD;GAEF,MAAM,WAAW,MAAM,MAAM,YAAY;IACvC,SAAS;KACP,cAAc;KACd,mBAAmB;IACrB;IACA,UAAU;IACV,QAAQ,WAAW;GACrB,CAAC;GACD,MAAM,WAAW,SAAS,QAAQ,IAAI,UAAU;GAChD,IACE,SAAS,UAAU,OACnB,SAAS,SAAS,OAClB,YACA,gBAAgB,eAChB;IACA,aAAa,IAAI,IAAI,UAAU,UAAU,CAAC,CAAC;IAC3C;GACF;GACA,OAAO;IACL,QAAQ,SAAS;IACjB,IAAI,SAAS;IACb,MAAM,MAAM,eAAe,QAAQ;IACnC,UAAU;IACV,YAAY,gBAAgB;GAC9B;EACF;CACF,QAAQ;EACN;CACF,UAAU;EACR,aAAa,KAAK;CACpB;AACF"}
|
package/dist/cjs/scan/index.cjs
CHANGED
|
@@ -1,31 +1,87 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
|
|
2
|
+
const require_scan_fetchText = require('./fetchText.cjs');
|
|
2
3
|
const require_scan_calculateScore = require('./calculateScore.cjs');
|
|
4
|
+
const require_scan_detection_technologySignatures = require('./detection/technologySignatures.cjs');
|
|
5
|
+
const require_scan_detection_url = require('./detection/url.cjs');
|
|
6
|
+
const require_scan_detection_detectTechnologies = require('./detection/detectTechnologies.cjs');
|
|
3
7
|
const require_scan_parseHtml = require('./parseHtml.cjs');
|
|
4
8
|
const require_scan_analyzeBundleContent = require('./analyzeBundleContent.cjs');
|
|
9
|
+
const require_scan_detection_localeCode = require('./detection/localeCode.cjs');
|
|
10
|
+
const require_scan_detection_detectRoutingStrategy = require('./detection/detectRoutingStrategy.cjs');
|
|
11
|
+
const require_scan_detection_classifyInternalLinks = require('./detection/classifyInternalLinks.cjs');
|
|
12
|
+
const require_scan_robots = require('./robots.cjs');
|
|
13
|
+
const require_scan_sitemap = require('./sitemap.cjs');
|
|
5
14
|
const require_scan_checks = require('./checks.cjs');
|
|
15
|
+
const require_scan_runScanChecks = require('./runScanChecks.cjs');
|
|
6
16
|
const require_scan_scanWebsite = require('./scanWebsite.cjs');
|
|
17
|
+
const require_scan_detection_checkDetails = require('./detection/checkDetails.cjs');
|
|
18
|
+
const require_scan_detection_localizedPages = require('./detection/localizedPages.cjs');
|
|
7
19
|
|
|
8
20
|
exports.analyzeBundleContent = require_scan_analyzeBundleContent.analyzeBundleContent;
|
|
9
21
|
exports.byteLength = require_scan_parseHtml.byteLength;
|
|
10
22
|
exports.checkBundleContent = require_scan_checks.checkBundleContent;
|
|
11
23
|
exports.checkCanonical = require_scan_checks.checkCanonical;
|
|
12
|
-
exports.
|
|
13
|
-
exports.
|
|
24
|
+
exports.checkHreflang = require_scan_checks.checkHreflang;
|
|
25
|
+
exports.checkHreflangReciprocity = require_scan_checks.checkHreflangReciprocity;
|
|
26
|
+
exports.checkHtmlDir = require_scan_checks.checkHtmlDir;
|
|
27
|
+
exports.checkHtmlLang = require_scan_checks.checkHtmlLang;
|
|
28
|
+
exports.checkInternalLinks = require_scan_checks.checkInternalLinks;
|
|
29
|
+
exports.checkLocaleConsistency = require_scan_checks.checkLocaleConsistency;
|
|
30
|
+
exports.checkOgLocale = require_scan_checks.checkOgLocale;
|
|
14
31
|
exports.checkRobots = require_scan_checks.checkRobots;
|
|
15
32
|
exports.checkSitemap = require_scan_checks.checkSitemap;
|
|
16
|
-
exports.
|
|
33
|
+
exports.checkXDefault = require_scan_checks.checkXDefault;
|
|
34
|
+
exports.classifyInternalLinks = require_scan_detection_classifyInternalLinks.classifyInternalLinks;
|
|
35
|
+
exports.collectSitemapEntries = require_scan_sitemap.collectSitemapEntries;
|
|
36
|
+
exports.detectRoutingStrategy = require_scan_detection_detectRoutingStrategy.detectRoutingStrategy;
|
|
37
|
+
exports.detectTechnologies = require_scan_detection_detectTechnologies.detectTechnologies;
|
|
38
|
+
exports.discoverSitemapUrls = require_scan_sitemap.discoverSitemapUrls;
|
|
17
39
|
exports.extractAnchors = require_scan_parseHtml.extractAnchors;
|
|
40
|
+
exports.extractCanonicalHref = require_scan_parseHtml.extractCanonicalHref;
|
|
18
41
|
exports.extractHreflangs = require_scan_parseHtml.extractHreflangs;
|
|
19
42
|
exports.extractHtmlDir = require_scan_parseHtml.extractHtmlDir;
|
|
20
43
|
exports.extractHtmlLang = require_scan_parseHtml.extractHtmlLang;
|
|
21
44
|
exports.extractMetaDescription = require_scan_parseHtml.extractMetaDescription;
|
|
22
45
|
exports.extractOgImage = require_scan_parseHtml.extractOgImage;
|
|
46
|
+
exports.extractOgLocale = require_scan_parseHtml.extractOgLocale;
|
|
47
|
+
exports.extractOgLocaleAlternates = require_scan_parseHtml.extractOgLocaleAlternates;
|
|
48
|
+
exports.extractResourceUrls = require_scan_parseHtml.extractResourceUrls;
|
|
23
49
|
exports.extractScriptUrls = require_scan_parseHtml.extractScriptUrls;
|
|
24
50
|
exports.extractTitle = require_scan_parseHtml.extractTitle;
|
|
25
51
|
exports.extractVisibleTextStrings = require_scan_parseHtml.extractVisibleTextStrings;
|
|
52
|
+
exports.fetchText = require_scan_fetchText.fetchText;
|
|
53
|
+
exports.findMatchingLocale = require_scan_detection_localeCode.findMatchingLocale;
|
|
54
|
+
exports.formatBundleAnalysis = require_scan_checks.formatBundleAnalysis;
|
|
26
55
|
exports.formatSize = require_scan_checks.formatSize;
|
|
56
|
+
exports.getCheckDetailLines = require_scan_detection_checkDetails.getCheckDetailLines;
|
|
57
|
+
exports.getDefaultSitemapUrls = require_scan_sitemap.getDefaultSitemapUrls;
|
|
58
|
+
exports.getFirstPathSegment = require_scan_detection_url.getFirstPathSegment;
|
|
59
|
+
exports.getLanguageCode = require_scan_detection_localeCode.getLanguageCode;
|
|
60
|
+
exports.getLocalizedPages = require_scan_detection_localizedPages.getLocalizedPages;
|
|
61
|
+
exports.getSiteDomain = require_scan_detection_url.getSiteDomain;
|
|
62
|
+
exports.getTechnologyGlobalNames = require_scan_detection_detectTechnologies.getTechnologyGlobalNames;
|
|
63
|
+
exports.getUrlLocale = require_scan_detection_detectRoutingStrategy.getUrlLocale;
|
|
27
64
|
exports.hasCanonical = require_scan_parseHtml.hasCanonical;
|
|
65
|
+
exports.isAbsoluteUrl = require_scan_detection_url.isAbsoluteUrl;
|
|
66
|
+
exports.isBaseLocalePage = require_scan_detection_localizedPages.isBaseLocalePage;
|
|
67
|
+
exports.isPathAllowedByRobots = require_scan_robots.isPathAllowedByRobots;
|
|
68
|
+
exports.isRightToLeftLocale = require_scan_detection_localeCode.isRightToLeftLocale;
|
|
69
|
+
exports.isSameLanguage = require_scan_detection_localeCode.isSameLanguage;
|
|
70
|
+
exports.isSameLocale = require_scan_detection_localeCode.isSameLocale;
|
|
71
|
+
exports.isValidLocaleCode = require_scan_detection_localeCode.isValidLocaleCode;
|
|
72
|
+
exports.isValidOpenGraphLocale = require_scan_detection_localeCode.isValidOpenGraphLocale;
|
|
73
|
+
exports.looksLikeLocaleCode = require_scan_detection_localeCode.looksLikeLocaleCode;
|
|
28
74
|
exports.mutateScore = require_scan_calculateScore.mutateScore;
|
|
75
|
+
exports.normalizeHostname = require_scan_detection_url.normalizeHostname;
|
|
76
|
+
exports.normalizeLocaleCode = require_scan_detection_localeCode.normalizeLocaleCode;
|
|
77
|
+
exports.normalizeUrl = require_scan_detection_url.normalizeUrl;
|
|
78
|
+
exports.parseRobots = require_scan_robots.parseRobots;
|
|
79
|
+
exports.parseSitemap = require_scan_sitemap.parseSitemap;
|
|
80
|
+
exports.parseUrl = require_scan_detection_url.parseUrl;
|
|
81
|
+
exports.runScanChecks = require_scan_runScanChecks.runScanChecks;
|
|
29
82
|
exports.scanWebsite = require_scan_scanWebsite.scanWebsite;
|
|
30
83
|
exports.scoreRecord = require_scan_calculateScore.scoreRecord;
|
|
84
|
+
exports.splitCheckDetails = require_scan_detection_checkDetails.splitCheckDetails;
|
|
85
|
+
exports.technologySignatures = require_scan_detection_technologySignatures.technologySignatures;
|
|
86
|
+
exports.toOpenGraphLocale = require_scan_detection_localeCode.toOpenGraphLocale;
|
|
31
87
|
exports.toScorePercent = require_scan_calculateScore.toScorePercent;
|
|
@@ -19,17 +19,17 @@ const readAttribute = (attributes, attributeName) => {
|
|
|
19
19
|
/** Extract the `lang` attribute of the `<html>` element, if present. */
|
|
20
20
|
const extractHtmlLang = (html) => {
|
|
21
21
|
const htmlTag = html.match(/<html\b([^>]*)>/i);
|
|
22
|
-
return htmlTag ? readAttribute(htmlTag[1], "lang") : void 0;
|
|
22
|
+
return htmlTag?.[1] ? readAttribute(htmlTag[1], "lang") : void 0;
|
|
23
23
|
};
|
|
24
24
|
/** Extract the `dir` attribute of the `<html>` element, if present. */
|
|
25
25
|
const extractHtmlDir = (html) => {
|
|
26
26
|
const htmlTag = html.match(/<html\b([^>]*)>/i);
|
|
27
|
-
return htmlTag ? readAttribute(htmlTag[1], "dir") : void 0;
|
|
27
|
+
return htmlTag?.[1] ? readAttribute(htmlTag[1], "dir") : void 0;
|
|
28
28
|
};
|
|
29
29
|
/** Extract the document `<title>` text. */
|
|
30
30
|
const extractTitle = (html) => {
|
|
31
31
|
const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
|
|
32
|
-
return match ? match[1].trim() : "";
|
|
32
|
+
return match?.[1] ? match[1].trim() : "";
|
|
33
33
|
};
|
|
34
34
|
/** Extract the `<meta name="description">` content. */
|
|
35
35
|
const extractMetaDescription = (html) => {
|
|
@@ -40,12 +40,28 @@ const extractMetaDescription = (html) => {
|
|
|
40
40
|
/** Extract the `<meta property="og:image">` content. */
|
|
41
41
|
const extractOgImage = (html) => {
|
|
42
42
|
const metas = html.match(/<meta\b[^>]*>/gi) ?? [];
|
|
43
|
-
for (const meta of metas) if (/property\s*=\s*("|')?og:image\1
|
|
43
|
+
for (const meta of metas) if (/property\s*=\s*("|')?og:image\1?[\s>/]/i.test(meta)) return readAttribute(meta, "content");
|
|
44
44
|
};
|
|
45
|
-
/**
|
|
46
|
-
const
|
|
47
|
-
|
|
45
|
+
/** Extract the `<meta property="og:locale">` content. */
|
|
46
|
+
const extractOgLocale = (html) => {
|
|
47
|
+
const metas = html.match(/<meta\b[^>]*>/gi) ?? [];
|
|
48
|
+
for (const meta of metas) if (/property\s*=\s*("|')?og:locale\1?[\s>/]/i.test(meta)) return readAttribute(meta, "content");
|
|
49
|
+
};
|
|
50
|
+
/** Extract every `<meta property="og:locale:alternate">` content. */
|
|
51
|
+
const extractOgLocaleAlternates = (html) => {
|
|
52
|
+
return (html.match(/<meta\b[^>]*>/gi) ?? []).flatMap((meta) => {
|
|
53
|
+
if (!/property\s*=\s*("|')?og:locale:alternate\1?[\s>/]/i.test(meta)) return [];
|
|
54
|
+
const content = readAttribute(meta, "content");
|
|
55
|
+
return content ? [content] : [];
|
|
56
|
+
});
|
|
57
|
+
};
|
|
58
|
+
/** Extract the `href` of the `<link rel="canonical">` element, if present. */
|
|
59
|
+
const extractCanonicalHref = (html) => {
|
|
60
|
+
const canonicalLink = (html.match(/<link\b[^>]*>/gi) ?? []).find((link) => /rel\s*=\s*("|')?canonical\1?/i.test(link));
|
|
61
|
+
return canonicalLink ? readAttribute(canonicalLink, "href") ?? "" : void 0;
|
|
48
62
|
};
|
|
63
|
+
/** Whether a `<link rel="canonical">` element is present. */
|
|
64
|
+
const hasCanonical = (html) => extractCanonicalHref(html) !== void 0;
|
|
49
65
|
/** Extract every `<link rel="alternate" hreflang="…" href="…">` element. */
|
|
50
66
|
const extractHreflangs = (html) => {
|
|
51
67
|
const links = html.match(/<link\b[^>]*>/gi) ?? [];
|
|
@@ -91,12 +107,15 @@ const extractAnchors = (html) => {
|
|
|
91
107
|
const anchorPattern = /<a\b([^>]*)>([\s\S]*?)<\/a>/gi;
|
|
92
108
|
let match = anchorPattern.exec(html);
|
|
93
109
|
while (match !== null) {
|
|
94
|
-
const
|
|
110
|
+
const rawAttrs = match[1] ?? "";
|
|
111
|
+
const rawContent = match[2] ?? "";
|
|
112
|
+
const href = readAttribute(rawAttrs, "href");
|
|
95
113
|
if (href) {
|
|
96
|
-
const text =
|
|
114
|
+
const text = rawContent.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
97
115
|
anchors.push({
|
|
98
116
|
href,
|
|
99
|
-
text
|
|
117
|
+
text,
|
|
118
|
+
hreflang: readAttribute(rawAttrs, "hreflang")
|
|
100
119
|
});
|
|
101
120
|
}
|
|
102
121
|
match = anchorPattern.exec(html);
|
|
@@ -104,6 +123,28 @@ const extractAnchors = (html) => {
|
|
|
104
123
|
return anchors;
|
|
105
124
|
};
|
|
106
125
|
/**
|
|
126
|
+
* Extract every resource URL referenced by the document (`<script src>`,
|
|
127
|
+
* `<link href>`, `<iframe src>`), used to fingerprint third-party services.
|
|
128
|
+
*
|
|
129
|
+
* @param html - The raw HTML document.
|
|
130
|
+
* @param baseUrl - Base URL used to resolve relative URLs.
|
|
131
|
+
* @returns Absolute, de-duplicated URLs.
|
|
132
|
+
*/
|
|
133
|
+
const extractResourceUrls = (html, baseUrl) => {
|
|
134
|
+
const urls = /* @__PURE__ */ new Set();
|
|
135
|
+
const tagPattern = /<(script|link|iframe)\b([^>]*)>/gi;
|
|
136
|
+
let match = tagPattern.exec(html);
|
|
137
|
+
while (match !== null) {
|
|
138
|
+
const attributeName = match[1]?.toLowerCase() === "link" ? "href" : "src";
|
|
139
|
+
const rawUrl = readAttribute(match[2] ?? "", attributeName);
|
|
140
|
+
if (rawUrl) try {
|
|
141
|
+
urls.add(new URL(rawUrl, baseUrl).href);
|
|
142
|
+
} catch {}
|
|
143
|
+
match = tagPattern.exec(html);
|
|
144
|
+
}
|
|
145
|
+
return Array.from(urls);
|
|
146
|
+
};
|
|
147
|
+
/**
|
|
107
148
|
* Extract visible text snippets from an HTML document (scripts, styles and
|
|
108
149
|
* tags stripped). Used to approximate the rendered content size without a DOM.
|
|
109
150
|
*/
|
|
@@ -114,11 +155,15 @@ const extractVisibleTextStrings = (html) => {
|
|
|
114
155
|
//#endregion
|
|
115
156
|
exports.byteLength = byteLength;
|
|
116
157
|
exports.extractAnchors = extractAnchors;
|
|
158
|
+
exports.extractCanonicalHref = extractCanonicalHref;
|
|
117
159
|
exports.extractHreflangs = extractHreflangs;
|
|
118
160
|
exports.extractHtmlDir = extractHtmlDir;
|
|
119
161
|
exports.extractHtmlLang = extractHtmlLang;
|
|
120
162
|
exports.extractMetaDescription = extractMetaDescription;
|
|
121
163
|
exports.extractOgImage = extractOgImage;
|
|
164
|
+
exports.extractOgLocale = extractOgLocale;
|
|
165
|
+
exports.extractOgLocaleAlternates = extractOgLocaleAlternates;
|
|
166
|
+
exports.extractResourceUrls = extractResourceUrls;
|
|
122
167
|
exports.extractScriptUrls = extractScriptUrls;
|
|
123
168
|
exports.extractTitle = extractTitle;
|
|
124
169
|
exports.extractVisibleTextStrings = extractVisibleTextStrings;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parseHtml.cjs","names":[],"sources":["../../../src/scan/parseHtml.ts"],"sourcesContent":["/**\n * Tiny dependency-free HTML extraction helpers.\n *\n * The hosted backend audit relies on Cheerio + a real browser, but the CLI scan\n * must stay dependency-light. These regex-based helpers cover the handful of\n * head/anchor signals the score needs. They are intentionally forgiving: when a\n * tag can't be parsed it is simply skipped rather than throwing.\n */\n\n/** Compute the UTF-8 byte length of a string in both Node and browser builds. */\nexport const byteLength = (text: string): number =>\n typeof Buffer !== 'undefined'\n ? Buffer.byteLength(text, 'utf-8')\n : new TextEncoder().encode(text).length;\n\n/** Read an attribute value off a single tag's attribute string. */\nconst readAttribute = (\n attributes: string,\n attributeName: string\n): string | undefined => {\n const match = attributes.match(\n new RegExp(`${attributeName}\\\\s*=\\\\s*(\"([^\"]*)\"|'([^']*)'|([^\\\\s>]+))`, 'i')\n );\n if (!match) return undefined;\n return match[2] ?? match[3] ?? match[4];\n};\n\n/** Extract the `lang` attribute of the `<html>` element, if present. */\nexport const extractHtmlLang = (html: string): string | undefined => {\n const htmlTag = html.match(/<html\\b([^>]*)>/i);\n return htmlTag ? readAttribute(htmlTag[1], 'lang') : undefined;\n};\n\n/** Extract the `dir` attribute of the `<html>` element, if present. */\nexport const extractHtmlDir = (html: string): string | undefined => {\n const htmlTag = html.match(/<html\\b([^>]*)>/i);\n return htmlTag ? readAttribute(htmlTag[1], 'dir') : undefined;\n};\n\n/** Extract the document `<title>` text. */\nexport const extractTitle = (html: string): string => {\n const match = html.match(/<title[^>]*>([\\s\\S]*?)<\\/title>/i);\n return match ? match[1].trim() : '';\n};\n\n/** Extract the `<meta name=\"description\">` content. */\nexport const extractMetaDescription = (html: string): string => {\n const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n for (const meta of metas) {\n if (/name\\s*=\\s*(\"|')?description\\1?/i.test(meta)) {\n return readAttribute(meta, 'content') ?? '';\n }\n }\n return '';\n};\n\n/** Extract the `<meta property=\"og:image\">` content. */\nexport const extractOgImage = (html: string): string | undefined => {\n const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n for (const meta of metas) {\n if (/property\\s*=\\s*(\"|')?og:image\\1?/i.test(meta)) {\n return readAttribute(meta, 'content');\n }\n }\n return undefined;\n};\n\n/** Whether a `<link rel=\"canonical\">` element is present. */\nexport const hasCanonical = (html: string): boolean => {\n const links = html.match(/<link\\b[^>]*>/gi) ?? [];\n return links.some((link) => /rel\\s*=\\s*(\"|')?canonical\\1?/i.test(link));\n};\n\n/** A parsed `<link rel=\"alternate\" hreflang>` element. */\nexport type HreflangLink = { hreflang: string; href: string };\n\n/** Extract every `<link rel=\"alternate\" hreflang=\"…\" href=\"…\">` element. */\nexport const extractHreflangs = (html: string): HreflangLink[] => {\n const links = html.match(/<link\\b[^>]*>/gi) ?? [];\n const result: HreflangLink[] = [];\n for (const link of links) {\n if (!/rel\\s*=\\s*(\"|')?alternate\\1?/i.test(link)) continue;\n const hreflang = readAttribute(link, 'hreflang');\n const href = readAttribute(link, 'href');\n if (hreflang && href) result.push({ hreflang, href });\n }\n return result;\n};\n\n/**\n * Extract every eagerly-loaded script URL: `<script src>`,\n * `<link rel=\"modulepreload\">` and `<link rel=\"preload\" as=\"script\">`.\n *\n * @param html - The raw HTML document.\n * @param baseUrl - Base URL used to resolve relative script URLs.\n * @returns Absolute, de-duplicated script URLs.\n */\nexport const extractScriptUrls = (html: string, baseUrl: string): string[] => {\n const urls = new Set<string>();\n\n const add = (raw: string | undefined) => {\n if (!raw) return;\n try {\n urls.add(new URL(raw, baseUrl).href);\n } catch {\n /* ignore malformed URLs */\n }\n };\n\n for (const script of html.match(/<script\\b[^>]*>/gi) ?? []) {\n add(readAttribute(script, 'src'));\n }\n\n for (const link of html.match(/<link\\b[^>]*>/gi) ?? []) {\n const rel = readAttribute(link, 'rel')?.toLowerCase();\n const as = readAttribute(link, 'as')?.toLowerCase();\n if (rel === 'modulepreload' || (rel === 'preload' && as === 'script')) {\n add(readAttribute(link, 'href'));\n }\n }\n\n return Array.from(urls);\n};\n\n/** A parsed `<a href>` anchor. */\nexport type Anchor = { href: string; text: string };\n\n/** Extract every `<a href=\"…\">text</a>` anchor from the document. */\nexport const extractAnchors = (html: string): Anchor[] => {\n const anchors: Anchor[] = [];\n const anchorPattern = /<a\\b([^>]*)>([\\s\\S]*?)<\\/a>/gi;\n let match = anchorPattern.exec(html);\n while (match !== null) {\n const href = readAttribute(match[1], 'href');\n if (href) {\n const text = match[2]\n .replace(/<[^>]+>/g, ' ')\n .replace(/\\s+/g, ' ')\n .trim();\n anchors.push({ href, text });\n }\n match = anchorPattern.exec(html);\n }\n return anchors;\n};\n\n/**\n * Extract visible text snippets from an HTML document (scripts, styles and\n * tags stripped). Used to approximate the rendered content size without a DOM.\n */\nexport const extractVisibleTextStrings = (html: string): string[] => {\n const withoutNonVisible = html\n .replace(/<script[\\s\\S]*?<\\/script>/gi, ' ')\n .replace(/<style[\\s\\S]*?<\\/style>/gi, ' ')\n .replace(/<noscript[\\s\\S]*?<\\/noscript>/gi, ' ')\n .replace(/<!--[\\s\\S]*?-->/g, ' ');\n\n return withoutNonVisible\n .replace(/<[^>]+>/g, '\\n')\n .split('\\n')\n .map((line) => line.replace(/\\s+/g, ' ').trim())\n .filter((line) => line.length > 1);\n};\n"],"mappings":";;;;;;;;;;;AAUA,MAAa,cAAc,SACzB,OAAO,WAAW,cACd,OAAO,WAAW,MAAM,OAAO,IAC/B,IAAI,YAAY,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;;AAGrC,MAAM,iBACJ,YACA,kBACuB;CACvB,MAAM,QAAQ,WAAW,MACvB,IAAI,OAAO,GAAG,cAAc,4CAA4C,GAAG,CAC7E;CACA,IAAI,CAAC,OAAO,OAAO;CACnB,OAAO,MAAM,MAAM,MAAM,MAAM,MAAM;AACvC;;AAGA,MAAa,mBAAmB,SAAqC;CACnE,MAAM,UAAU,KAAK,MAAM,kBAAkB;CAC7C,OAAO,UAAU,cAAc,QAAQ,IAAI,MAAM,IAAI;AACvD;;AAGA,MAAa,kBAAkB,SAAqC;CAClE,MAAM,UAAU,KAAK,MAAM,kBAAkB;CAC7C,OAAO,UAAU,cAAc,QAAQ,IAAI,KAAK,IAAI;AACtD;;AAGA,MAAa,gBAAgB,SAAyB;CACpD,MAAM,QAAQ,KAAK,MAAM,kCAAkC;CAC3D,OAAO,QAAQ,MAAM,EAAE,CAAC,KAAK,IAAI;AACnC;;AAGA,MAAa,0BAA0B,SAAyB;CAC9D,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,KAAK,MAAM,QAAQ,OACjB,IAAI,mCAAmC,KAAK,IAAI,GAC9C,OAAO,cAAc,MAAM,SAAS,KAAK;CAG7C,OAAO;AACT;;AAGA,MAAa,kBAAkB,SAAqC;CAClE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,KAAK,MAAM,QAAQ,OACjB,IAAI,oCAAoC,KAAK,IAAI,GAC/C,OAAO,cAAc,MAAM,SAAS;AAI1C;;AAGA,MAAa,gBAAgB,SAA0B;CAErD,QADc,KAAK,MAAM,iBAAiB,KAAK,CAAC,EACpC,CAAC,MAAM,SAAS,gCAAgC,KAAK,IAAI,CAAC;AACxE;;AAMA,MAAa,oBAAoB,SAAiC;CAChE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,MAAM,SAAyB,CAAC;CAChC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,gCAAgC,KAAK,IAAI,GAAG;EACjD,MAAM,WAAW,cAAc,MAAM,UAAU;EAC/C,MAAM,OAAO,cAAc,MAAM,MAAM;EACvC,IAAI,YAAY,MAAM,OAAO,KAAK;GAAE;GAAU;EAAK,CAAC;CACtD;CACA,OAAO;AACT;;;;;;;;;AAUA,MAAa,qBAAqB,MAAc,YAA8B;CAC5E,MAAM,uBAAO,IAAI,IAAY;CAE7B,MAAM,OAAO,QAA4B;EACvC,IAAI,CAAC,KAAK;EACV,IAAI;GACF,KAAK,IAAI,IAAI,IAAI,KAAK,OAAO,CAAC,CAAC,IAAI;EACrC,QAAQ,CAER;CACF;CAEA,KAAK,MAAM,UAAU,KAAK,MAAM,mBAAmB,KAAK,CAAC,GACvD,IAAI,cAAc,QAAQ,KAAK,CAAC;CAGlC,KAAK,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC,GAAG;EACtD,MAAM,MAAM,cAAc,MAAM,KAAK,CAAC,EAAE,YAAY;EACpD,MAAM,KAAK,cAAc,MAAM,IAAI,CAAC,EAAE,YAAY;EAClD,IAAI,QAAQ,mBAAoB,QAAQ,aAAa,OAAO,UAC1D,IAAI,cAAc,MAAM,MAAM,CAAC;CAEnC;CAEA,OAAO,MAAM,KAAK,IAAI;AACxB;;AAMA,MAAa,kBAAkB,SAA2B;CACxD,MAAM,UAAoB,CAAC;CAC3B,MAAM,gBAAgB;CACtB,IAAI,QAAQ,cAAc,KAAK,IAAI;CACnC,OAAO,UAAU,MAAM;EACrB,MAAM,OAAO,cAAc,MAAM,IAAI,MAAM;EAC3C,IAAI,MAAM;GACR,MAAM,OAAO,MAAM,EAAE,CAClB,QAAQ,YAAY,GAAG,CAAC,CACxB,QAAQ,QAAQ,GAAG,CAAC,CACpB,KAAK;GACR,QAAQ,KAAK;IAAE;IAAM;GAAK,CAAC;EAC7B;EACA,QAAQ,cAAc,KAAK,IAAI;CACjC;CACA,OAAO;AACT;;;;;AAMA,MAAa,6BAA6B,SAA2B;CAOnE,OAN0B,KACvB,QAAQ,+BAA+B,GAAG,CAAC,CAC3C,QAAQ,6BAA6B,GAAG,CAAC,CACzC,QAAQ,mCAAmC,GAAG,CAAC,CAC/C,QAAQ,oBAAoB,GAER,CAAC,CACrB,QAAQ,YAAY,IAAI,CAAC,CACzB,MAAM,IAAI,CAAC,CACX,KAAK,SAAS,KAAK,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAC/C,QAAQ,SAAS,KAAK,SAAS,CAAC;AACrC"}
|
|
1
|
+
{"version":3,"file":"parseHtml.cjs","names":[],"sources":["../../../src/scan/parseHtml.ts"],"sourcesContent":["/**\n * Tiny dependency-free HTML extraction helpers.\n *\n * The hosted backend audit relies on Cheerio + a real browser, but the CLI scan\n * must stay dependency-light. These regex-based helpers cover the handful of\n * head/anchor signals the score needs. They are intentionally forgiving: when a\n * tag can't be parsed it is simply skipped rather than throwing.\n */\n\n/** Compute the UTF-8 byte length of a string in both Node and browser builds. */\nexport const byteLength = (text: string): number =>\n typeof Buffer !== 'undefined'\n ? Buffer.byteLength(text, 'utf-8')\n : new TextEncoder().encode(text).length;\n\n/** Read an attribute value off a single tag's attribute string. */\nconst readAttribute = (\n attributes: string,\n attributeName: string\n): string | undefined => {\n const match = attributes.match(\n new RegExp(`${attributeName}\\\\s*=\\\\s*(\"([^\"]*)\"|'([^']*)'|([^\\\\s>]+))`, 'i')\n );\n if (!match) return undefined;\n return match[2] ?? match[3] ?? match[4];\n};\n\n/** Extract the `lang` attribute of the `<html>` element, if present. */\nexport const extractHtmlLang = (html: string): string | undefined => {\n const htmlTag = html.match(/<html\\b([^>]*)>/i);\n return htmlTag?.[1] ? readAttribute(htmlTag[1], 'lang') : undefined;\n};\n\n/** Extract the `dir` attribute of the `<html>` element, if present. */\nexport const extractHtmlDir = (html: string): string | undefined => {\n const htmlTag = html.match(/<html\\b([^>]*)>/i);\n return htmlTag?.[1] ? readAttribute(htmlTag[1], 'dir') : undefined;\n};\n\n/** Extract the document `<title>` text. */\nexport const extractTitle = (html: string): string => {\n const match = html.match(/<title[^>]*>([\\s\\S]*?)<\\/title>/i);\n return match?.[1] ? match[1].trim() : '';\n};\n\n/** Extract the `<meta name=\"description\">` content. */\nexport const extractMetaDescription = (html: string): string => {\n const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n for (const meta of metas) {\n if (/name\\s*=\\s*(\"|')?description\\1?/i.test(meta)) {\n return readAttribute(meta, 'content') ?? '';\n }\n }\n return '';\n};\n\n/** Extract the `<meta property=\"og:image\">` content. */\nexport const extractOgImage = (html: string): string | undefined => {\n const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n for (const meta of metas) {\n if (/property\\s*=\\s*(\"|')?og:image\\1?[\\s>/]/i.test(meta)) {\n return readAttribute(meta, 'content');\n }\n }\n return undefined;\n};\n\n/** Extract the `<meta property=\"og:locale\">` content. */\nexport const extractOgLocale = (html: string): string | undefined => {\n const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n for (const meta of metas) {\n if (/property\\s*=\\s*(\"|')?og:locale\\1?[\\s>/]/i.test(meta)) {\n return readAttribute(meta, 'content');\n }\n }\n return undefined;\n};\n\n/** Extract every `<meta property=\"og:locale:alternate\">` content. */\nexport const extractOgLocaleAlternates = (html: string): string[] => {\n const metas = html.match(/<meta\\b[^>]*>/gi) ?? [];\n return metas.flatMap((meta) => {\n if (!/property\\s*=\\s*(\"|')?og:locale:alternate\\1?[\\s>/]/i.test(meta)) {\n return [];\n }\n const content = readAttribute(meta, 'content');\n return content ? [content] : [];\n });\n};\n\n/** Extract the `href` of the `<link rel=\"canonical\">` element, if present. */\nexport const extractCanonicalHref = (html: string): string | undefined => {\n const links = html.match(/<link\\b[^>]*>/gi) ?? [];\n const canonicalLink = links.find((link) =>\n /rel\\s*=\\s*(\"|')?canonical\\1?/i.test(link)\n );\n return canonicalLink\n ? (readAttribute(canonicalLink, 'href') ?? '')\n : undefined;\n};\n\n/** Whether a `<link rel=\"canonical\">` element is present. */\nexport const hasCanonical = (html: string): boolean =>\n extractCanonicalHref(html) !== undefined;\n\n/** A parsed `<link rel=\"alternate\" hreflang>` element. */\nexport type HreflangLink = { hreflang: string; href: string };\n\n/** Extract every `<link rel=\"alternate\" hreflang=\"…\" href=\"…\">` element. */\nexport const extractHreflangs = (html: string): HreflangLink[] => {\n const links = html.match(/<link\\b[^>]*>/gi) ?? [];\n const result: HreflangLink[] = [];\n for (const link of links) {\n if (!/rel\\s*=\\s*(\"|')?alternate\\1?/i.test(link)) continue;\n const hreflang = readAttribute(link, 'hreflang');\n const href = readAttribute(link, 'href');\n if (hreflang && href) result.push({ hreflang, href });\n }\n return result;\n};\n\n/**\n * Extract every eagerly-loaded script URL: `<script src>`,\n * `<link rel=\"modulepreload\">` and `<link rel=\"preload\" as=\"script\">`.\n *\n * @param html - The raw HTML document.\n * @param baseUrl - Base URL used to resolve relative script URLs.\n * @returns Absolute, de-duplicated script URLs.\n */\nexport const extractScriptUrls = (html: string, baseUrl: string): string[] => {\n const urls = new Set<string>();\n\n const add = (raw: string | undefined) => {\n if (!raw) return;\n try {\n urls.add(new URL(raw, baseUrl).href);\n } catch {\n /* ignore malformed URLs */\n }\n };\n\n for (const script of html.match(/<script\\b[^>]*>/gi) ?? []) {\n add(readAttribute(script, 'src'));\n }\n\n for (const link of html.match(/<link\\b[^>]*>/gi) ?? []) {\n const rel = readAttribute(link, 'rel')?.toLowerCase();\n const as = readAttribute(link, 'as')?.toLowerCase();\n if (rel === 'modulepreload' || (rel === 'preload' && as === 'script')) {\n add(readAttribute(link, 'href'));\n }\n }\n\n return Array.from(urls);\n};\n\n/** A parsed `<a href>` anchor. */\nexport type Anchor = {\n href: string;\n text: string;\n /** `hreflang` attribute, set on language-switcher links. */\n hreflang?: string;\n};\n\n/** Extract every `<a href=\"…\">text</a>` anchor from the document. */\nexport const extractAnchors = (html: string): Anchor[] => {\n const anchors: Anchor[] = [];\n const anchorPattern = /<a\\b([^>]*)>([\\s\\S]*?)<\\/a>/gi;\n let match = anchorPattern.exec(html);\n while (match !== null) {\n const rawAttrs = match[1] ?? '';\n const rawContent = match[2] ?? '';\n const href = readAttribute(rawAttrs, 'href');\n if (href) {\n const text = rawContent\n .replace(/<[^>]+>/g, ' ')\n .replace(/\\s+/g, ' ')\n .trim();\n anchors.push({\n href,\n text,\n hreflang: readAttribute(rawAttrs, 'hreflang'),\n });\n }\n match = anchorPattern.exec(html);\n }\n return anchors;\n};\n\n/**\n * Extract every resource URL referenced by the document (`<script src>`,\n * `<link href>`, `<iframe src>`), used to fingerprint third-party services.\n *\n * @param html - The raw HTML document.\n * @param baseUrl - Base URL used to resolve relative URLs.\n * @returns Absolute, de-duplicated URLs.\n */\nexport const extractResourceUrls = (\n html: string,\n baseUrl: string\n): string[] => {\n const urls = new Set<string>();\n const tagPattern = /<(script|link|iframe)\\b([^>]*)>/gi;\n let match = tagPattern.exec(html);\n while (match !== null) {\n const attributeName = match[1]?.toLowerCase() === 'link' ? 'href' : 'src';\n const rawUrl = readAttribute(match[2] ?? '', attributeName);\n if (rawUrl) {\n try {\n urls.add(new URL(rawUrl, baseUrl).href);\n } catch {\n /* ignore malformed URLs */\n }\n }\n match = tagPattern.exec(html);\n }\n return Array.from(urls);\n};\n\n/**\n * Extract visible text snippets from an HTML document (scripts, styles and\n * tags stripped). Used to approximate the rendered content size without a DOM.\n */\nexport const extractVisibleTextStrings = (html: string): string[] => {\n const withoutNonVisible = html\n .replace(/<script[\\s\\S]*?<\\/script>/gi, ' ')\n .replace(/<style[\\s\\S]*?<\\/style>/gi, ' ')\n .replace(/<noscript[\\s\\S]*?<\\/noscript>/gi, ' ')\n .replace(/<!--[\\s\\S]*?-->/g, ' ');\n\n return withoutNonVisible\n .replace(/<[^>]+>/g, '\\n')\n .split('\\n')\n .map((line) => line.replace(/\\s+/g, ' ').trim())\n .filter((line) => line.length > 1);\n};\n"],"mappings":";;;;;;;;;;;AAUA,MAAa,cAAc,SACzB,OAAO,WAAW,cACd,OAAO,WAAW,MAAM,OAAO,IAC/B,IAAI,YAAY,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;;AAGrC,MAAM,iBACJ,YACA,kBACuB;CACvB,MAAM,QAAQ,WAAW,MACvB,IAAI,OAAO,GAAG,cAAc,4CAA4C,GAAG,CAC7E;CACA,IAAI,CAAC,OAAO,OAAO;CACnB,OAAO,MAAM,MAAM,MAAM,MAAM,MAAM;AACvC;;AAGA,MAAa,mBAAmB,SAAqC;CACnE,MAAM,UAAU,KAAK,MAAM,kBAAkB;CAC7C,OAAO,UAAU,KAAK,cAAc,QAAQ,IAAI,MAAM,IAAI;AAC5D;;AAGA,MAAa,kBAAkB,SAAqC;CAClE,MAAM,UAAU,KAAK,MAAM,kBAAkB;CAC7C,OAAO,UAAU,KAAK,cAAc,QAAQ,IAAI,KAAK,IAAI;AAC3D;;AAGA,MAAa,gBAAgB,SAAyB;CACpD,MAAM,QAAQ,KAAK,MAAM,kCAAkC;CAC3D,OAAO,QAAQ,KAAK,MAAM,EAAE,CAAC,KAAK,IAAI;AACxC;;AAGA,MAAa,0BAA0B,SAAyB;CAC9D,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,KAAK,MAAM,QAAQ,OACjB,IAAI,mCAAmC,KAAK,IAAI,GAC9C,OAAO,cAAc,MAAM,SAAS,KAAK;CAG7C,OAAO;AACT;;AAGA,MAAa,kBAAkB,SAAqC;CAClE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,KAAK,MAAM,QAAQ,OACjB,IAAI,0CAA0C,KAAK,IAAI,GACrD,OAAO,cAAc,MAAM,SAAS;AAI1C;;AAGA,MAAa,mBAAmB,SAAqC;CACnE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,KAAK,MAAM,QAAQ,OACjB,IAAI,2CAA2C,KAAK,IAAI,GACtD,OAAO,cAAc,MAAM,SAAS;AAI1C;;AAGA,MAAa,6BAA6B,SAA2B;CAEnE,QADc,KAAK,MAAM,iBAAiB,KAAK,CAAC,EACpC,CAAC,SAAS,SAAS;EAC7B,IAAI,CAAC,qDAAqD,KAAK,IAAI,GACjE,OAAO,CAAC;EAEV,MAAM,UAAU,cAAc,MAAM,SAAS;EAC7C,OAAO,UAAU,CAAC,OAAO,IAAI,CAAC;CAChC,CAAC;AACH;;AAGA,MAAa,wBAAwB,SAAqC;CAExE,MAAM,iBADQ,KAAK,MAAM,iBAAiB,KAAK,CAAC,EACrB,CAAC,MAAM,SAChC,gCAAgC,KAAK,IAAI,CAC3C;CACA,OAAO,gBACF,cAAc,eAAe,MAAM,KAAK,KACzC;AACN;;AAGA,MAAa,gBAAgB,SAC3B,qBAAqB,IAAI,MAAM;;AAMjC,MAAa,oBAAoB,SAAiC;CAChE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC;CAChD,MAAM,SAAyB,CAAC;CAChC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,gCAAgC,KAAK,IAAI,GAAG;EACjD,MAAM,WAAW,cAAc,MAAM,UAAU;EAC/C,MAAM,OAAO,cAAc,MAAM,MAAM;EACvC,IAAI,YAAY,MAAM,OAAO,KAAK;GAAE;GAAU;EAAK,CAAC;CACtD;CACA,OAAO;AACT;;;;;;;;;AAUA,MAAa,qBAAqB,MAAc,YAA8B;CAC5E,MAAM,uBAAO,IAAI,IAAY;CAE7B,MAAM,OAAO,QAA4B;EACvC,IAAI,CAAC,KAAK;EACV,IAAI;GACF,KAAK,IAAI,IAAI,IAAI,KAAK,OAAO,CAAC,CAAC,IAAI;EACrC,QAAQ,CAER;CACF;CAEA,KAAK,MAAM,UAAU,KAAK,MAAM,mBAAmB,KAAK,CAAC,GACvD,IAAI,cAAc,QAAQ,KAAK,CAAC;CAGlC,KAAK,MAAM,QAAQ,KAAK,MAAM,iBAAiB,KAAK,CAAC,GAAG;EACtD,MAAM,MAAM,cAAc,MAAM,KAAK,CAAC,EAAE,YAAY;EACpD,MAAM,KAAK,cAAc,MAAM,IAAI,CAAC,EAAE,YAAY;EAClD,IAAI,QAAQ,mBAAoB,QAAQ,aAAa,OAAO,UAC1D,IAAI,cAAc,MAAM,MAAM,CAAC;CAEnC;CAEA,OAAO,MAAM,KAAK,IAAI;AACxB;;AAWA,MAAa,kBAAkB,SAA2B;CACxD,MAAM,UAAoB,CAAC;CAC3B,MAAM,gBAAgB;CACtB,IAAI,QAAQ,cAAc,KAAK,IAAI;CACnC,OAAO,UAAU,MAAM;EACrB,MAAM,WAAW,MAAM,MAAM;EAC7B,MAAM,aAAa,MAAM,MAAM;EAC/B,MAAM,OAAO,cAAc,UAAU,MAAM;EAC3C,IAAI,MAAM;GACR,MAAM,OAAO,WACV,QAAQ,YAAY,GAAG,CAAC,CACxB,QAAQ,QAAQ,GAAG,CAAC,CACpB,KAAK;GACR,QAAQ,KAAK;IACX;IACA;IACA,UAAU,cAAc,UAAU,UAAU;GAC9C,CAAC;EACH;EACA,QAAQ,cAAc,KAAK,IAAI;CACjC;CACA,OAAO;AACT;;;;;;;;;AAUA,MAAa,uBACX,MACA,YACa;CACb,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,aAAa;CACnB,IAAI,QAAQ,WAAW,KAAK,IAAI;CAChC,OAAO,UAAU,MAAM;EACrB,MAAM,gBAAgB,MAAM,EAAE,EAAE,YAAY,MAAM,SAAS,SAAS;EACpE,MAAM,SAAS,cAAc,MAAM,MAAM,IAAI,aAAa;EAC1D,IAAI,QACF,IAAI;GACF,KAAK,IAAI,IAAI,IAAI,QAAQ,OAAO,CAAC,CAAC,IAAI;EACxC,QAAQ,CAER;EAEF,QAAQ,WAAW,KAAK,IAAI;CAC9B;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;;;;;AAMA,MAAa,6BAA6B,SAA2B;CAOnE,OAN0B,KACvB,QAAQ,+BAA+B,GAAG,CAAC,CAC3C,QAAQ,6BAA6B,GAAG,CAAC,CACzC,QAAQ,mCAAmC,GAAG,CAAC,CAC/C,QAAQ,oBAAoB,GAER,CAAC,CACrB,QAAQ,YAAY,IAAI,CAAC,CACzB,MAAM,IAAI,CAAC,CACX,KAAK,SAAS,KAAK,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAC/C,QAAQ,SAAS,KAAK,SAAS,CAAC;AACrC"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
|
|
2
|
+
//#region src/scan/robots.ts
|
|
3
|
+
/**
|
|
4
|
+
* Parse a `robots.txt`, keeping the rules Googlebot would follow: the
|
|
5
|
+
* `googlebot` group when one exists, the `*` group otherwise.
|
|
6
|
+
*/
|
|
7
|
+
const parseRobots = (content) => {
|
|
8
|
+
const groups = [];
|
|
9
|
+
const sitemapUrls = [];
|
|
10
|
+
let currentGroup;
|
|
11
|
+
let isReadingUserAgents = false;
|
|
12
|
+
for (const rawLine of content.split(/\r?\n/)) {
|
|
13
|
+
const line = rawLine.replace(/#.*$/, "").trim();
|
|
14
|
+
const separatorIndex = line.indexOf(":");
|
|
15
|
+
if (separatorIndex === -1) continue;
|
|
16
|
+
const directive = line.slice(0, separatorIndex).trim().toLowerCase();
|
|
17
|
+
const value = line.slice(separatorIndex + 1).trim();
|
|
18
|
+
if (directive === "sitemap") {
|
|
19
|
+
if (value) sitemapUrls.push(value);
|
|
20
|
+
continue;
|
|
21
|
+
}
|
|
22
|
+
if (directive === "user-agent") {
|
|
23
|
+
if (!isReadingUserAgents || !currentGroup) {
|
|
24
|
+
currentGroup = {
|
|
25
|
+
userAgents: [],
|
|
26
|
+
disallowedPaths: [],
|
|
27
|
+
allowedPaths: []
|
|
28
|
+
};
|
|
29
|
+
groups.push(currentGroup);
|
|
30
|
+
}
|
|
31
|
+
currentGroup.userAgents.push(value.toLowerCase());
|
|
32
|
+
isReadingUserAgents = true;
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
isReadingUserAgents = false;
|
|
36
|
+
if (!currentGroup || !value) continue;
|
|
37
|
+
if (directive === "disallow") currentGroup.disallowedPaths.push(value);
|
|
38
|
+
if (directive === "allow") currentGroup.allowedPaths.push(value);
|
|
39
|
+
}
|
|
40
|
+
const googlebotGroups = groups.filter(({ userAgents }) => userAgents.includes("googlebot"));
|
|
41
|
+
const applicableGroups = googlebotGroups.length > 0 ? googlebotGroups : groups.filter(({ userAgents }) => userAgents.includes("*"));
|
|
42
|
+
return {
|
|
43
|
+
disallowedPaths: applicableGroups.flatMap(({ disallowedPaths }) => disallowedPaths),
|
|
44
|
+
allowedPaths: applicableGroups.flatMap(({ allowedPaths }) => allowedPaths),
|
|
45
|
+
sitemapUrls
|
|
46
|
+
};
|
|
47
|
+
};
|
|
48
|
+
/** Convert a robots.txt path pattern (`*`, `$`) into a RegExp. */
|
|
49
|
+
const robotsPatternToRegExp = (pattern) => new RegExp(`^${pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\\\$$/, "$")}`);
|
|
50
|
+
/**
|
|
51
|
+
* Whether Googlebot may crawl `pathWithSearch` (path + query string) under the
|
|
52
|
+
* parsed rules. Follows Google's precedence: the longest matching rule wins,
|
|
53
|
+
* `Allow` wins ties.
|
|
54
|
+
*/
|
|
55
|
+
const isPathAllowedByRobots = (pathWithSearch, robots) => {
|
|
56
|
+
const longestMatch = (patterns) => patterns.reduce((longest, pattern) => robotsPatternToRegExp(pattern).test(pathWithSearch) ? Math.max(longest, pattern.length) : longest, -1);
|
|
57
|
+
const disallowLength = longestMatch(robots.disallowedPaths);
|
|
58
|
+
if (disallowLength === -1) return true;
|
|
59
|
+
return longestMatch(robots.allowedPaths) >= disallowLength;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
//#endregion
|
|
63
|
+
exports.isPathAllowedByRobots = isPathAllowedByRobots;
|
|
64
|
+
exports.parseRobots = parseRobots;
|
|
65
|
+
//# sourceMappingURL=robots.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"robots.cjs","names":[],"sources":["../../../src/scan/robots.ts"],"sourcesContent":["/** Rules of a `robots.txt` that apply to Google's crawler. */\nexport type ParsedRobots = {\n /** `Disallow` paths of the group applying to Googlebot (or `*`). */\n disallowedPaths: string[];\n /** `Allow` paths of the same group. */\n allowedPaths: string[];\n /** Absolute URLs declared with `Sitemap:` directives. */\n sitemapUrls: string[];\n};\n\ntype RobotsGroup = {\n userAgents: string[];\n disallowedPaths: string[];\n allowedPaths: string[];\n};\n\n/**\n * Parse a `robots.txt`, keeping the rules Googlebot would follow: the\n * `googlebot` group when one exists, the `*` group otherwise.\n */\nexport const parseRobots = (content: string): ParsedRobots => {\n const groups: RobotsGroup[] = [];\n const sitemapUrls: string[] = [];\n let currentGroup: RobotsGroup | undefined;\n let isReadingUserAgents = false;\n\n for (const rawLine of content.split(/\\r?\\n/)) {\n const line = rawLine.replace(/#.*$/, '').trim();\n const separatorIndex = line.indexOf(':');\n if (separatorIndex === -1) continue;\n\n const directive = line.slice(0, separatorIndex).trim().toLowerCase();\n const value = line.slice(separatorIndex + 1).trim();\n\n if (directive === 'sitemap') {\n if (value) sitemapUrls.push(value);\n continue;\n }\n\n if (directive === 'user-agent') {\n if (!isReadingUserAgents || !currentGroup) {\n currentGroup = {\n userAgents: [],\n disallowedPaths: [],\n allowedPaths: [],\n };\n groups.push(currentGroup);\n }\n currentGroup.userAgents.push(value.toLowerCase());\n isReadingUserAgents = true;\n continue;\n }\n\n isReadingUserAgents = false;\n if (!currentGroup || !value) continue;\n if (directive === 'disallow') currentGroup.disallowedPaths.push(value);\n if (directive === 'allow') currentGroup.allowedPaths.push(value);\n }\n\n const googlebotGroups = groups.filter(({ userAgents }) =>\n userAgents.includes('googlebot')\n );\n const applicableGroups =\n googlebotGroups.length > 0\n ? googlebotGroups\n : groups.filter(({ userAgents }) => userAgents.includes('*'));\n\n return {\n disallowedPaths: applicableGroups.flatMap(\n ({ disallowedPaths }) => disallowedPaths\n ),\n allowedPaths: applicableGroups.flatMap(({ allowedPaths }) => allowedPaths),\n sitemapUrls,\n };\n};\n\n/** Convert a robots.txt path pattern (`*`, `$`) into a RegExp. */\nconst robotsPatternToRegExp = (pattern: string): RegExp =>\n new RegExp(\n `^${pattern\n .replace(/[.+?^${}()|[\\]\\\\]/g, '\\\\$&')\n .replace(/\\*/g, '.*')\n .replace(/\\\\\\$$/, '$')}`\n );\n\n/**\n * Whether Googlebot may crawl `pathWithSearch` (path + query string) under the\n * parsed rules. Follows Google's precedence: the longest matching rule wins,\n * `Allow` wins ties.\n */\nexport const isPathAllowedByRobots = (\n pathWithSearch: string,\n robots: Pick<ParsedRobots, 'disallowedPaths' | 'allowedPaths'>\n): boolean => {\n const longestMatch = (patterns: string[]): number =>\n patterns.reduce(\n (longest, pattern) =>\n robotsPatternToRegExp(pattern).test(pathWithSearch)\n ? Math.max(longest, pattern.length)\n : longest,\n -1\n );\n\n const disallowLength = longestMatch(robots.disallowedPaths);\n if (disallowLength === -1) return true;\n return longestMatch(robots.allowedPaths) >= disallowLength;\n};\n"],"mappings":";;;;;;AAoBA,MAAa,eAAe,YAAkC;CAC5D,MAAM,SAAwB,CAAC;CAC/B,MAAM,cAAwB,CAAC;CAC/B,IAAI;CACJ,IAAI,sBAAsB;CAE1B,KAAK,MAAM,WAAW,QAAQ,MAAM,OAAO,GAAG;EAC5C,MAAM,OAAO,QAAQ,QAAQ,QAAQ,EAAE,CAAC,CAAC,KAAK;EAC9C,MAAM,iBAAiB,KAAK,QAAQ,GAAG;EACvC,IAAI,mBAAmB,IAAI;EAE3B,MAAM,YAAY,KAAK,MAAM,GAAG,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,YAAY;EACnE,MAAM,QAAQ,KAAK,MAAM,iBAAiB,CAAC,CAAC,CAAC,KAAK;EAElD,IAAI,cAAc,WAAW;GAC3B,IAAI,OAAO,YAAY,KAAK,KAAK;GACjC;EACF;EAEA,IAAI,cAAc,cAAc;GAC9B,IAAI,CAAC,uBAAuB,CAAC,cAAc;IACzC,eAAe;KACb,YAAY,CAAC;KACb,iBAAiB,CAAC;KAClB,cAAc,CAAC;IACjB;IACA,OAAO,KAAK,YAAY;GAC1B;GACA,aAAa,WAAW,KAAK,MAAM,YAAY,CAAC;GAChD,sBAAsB;GACtB;EACF;EAEA,sBAAsB;EACtB,IAAI,CAAC,gBAAgB,CAAC,OAAO;EAC7B,IAAI,cAAc,YAAY,aAAa,gBAAgB,KAAK,KAAK;EACrE,IAAI,cAAc,SAAS,aAAa,aAAa,KAAK,KAAK;CACjE;CAEA,MAAM,kBAAkB,OAAO,QAAQ,EAAE,iBACvC,WAAW,SAAS,WAAW,CACjC;CACA,MAAM,mBACJ,gBAAgB,SAAS,IACrB,kBACA,OAAO,QAAQ,EAAE,iBAAiB,WAAW,SAAS,GAAG,CAAC;CAEhE,OAAO;EACL,iBAAiB,iBAAiB,SAC/B,EAAE,sBAAsB,eAC3B;EACA,cAAc,iBAAiB,SAAS,EAAE,mBAAmB,YAAY;EACzE;CACF;AACF;;AAGA,MAAM,yBAAyB,YAC7B,IAAI,OACF,IAAI,QACD,QAAQ,sBAAsB,MAAM,CAAC,CACrC,QAAQ,OAAO,IAAI,CAAC,CACpB,QAAQ,SAAS,GAAG,GACzB;;;;;;AAOF,MAAa,yBACX,gBACA,WACY;CACZ,MAAM,gBAAgB,aACpB,SAAS,QACN,SAAS,YACR,sBAAsB,OAAO,CAAC,CAAC,KAAK,cAAc,IAC9C,KAAK,IAAI,SAAS,QAAQ,MAAM,IAChC,SACN,EACF;CAEF,MAAM,iBAAiB,aAAa,OAAO,eAAe;CAC1D,IAAI,mBAAmB,IAAI,OAAO;CAClC,OAAO,aAAa,OAAO,YAAY,KAAK;AAC9C"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
|
|
2
|
+
const require_scan_detection_url = require('./detection/url.cjs');
|
|
3
|
+
const require_scan_detection_detectTechnologies = require('./detection/detectTechnologies.cjs');
|
|
4
|
+
const require_scan_parseHtml = require('./parseHtml.cjs');
|
|
5
|
+
const require_scan_detection_detectRoutingStrategy = require('./detection/detectRoutingStrategy.cjs');
|
|
6
|
+
const require_scan_checks = require('./checks.cjs');
|
|
7
|
+
|
|
8
|
+
//#region src/scan/runScanChecks.ts
|
|
9
|
+
/** Extract title, description and absolute preview image. */
|
|
10
|
+
const extractPageMetadata = (html, targetUrl) => {
|
|
11
|
+
const ogImage = require_scan_parseHtml.extractOgImage(html);
|
|
12
|
+
return {
|
|
13
|
+
title: require_scan_parseHtml.extractTitle(html),
|
|
14
|
+
description: require_scan_parseHtml.extractMetaDescription(html),
|
|
15
|
+
image: ogImage ? require_scan_detection_url.parseUrl(ogImage, targetUrl)?.href ?? ogImage : ""
|
|
16
|
+
};
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* Run every i18n/SEO check on an already loaded page. Single implementation
|
|
20
|
+
* shared by the `intlayer scan` CLI and the hosted audit (`/api/scan`), which
|
|
21
|
+
* only differ in how they load the page.
|
|
22
|
+
*
|
|
23
|
+
* @param input - The loaded page (HTML, chunks, size, runtime signals).
|
|
24
|
+
* @param emit - Receives every check result as soon as it is produced.
|
|
25
|
+
* @param options - Network options and progress callbacks.
|
|
26
|
+
*/
|
|
27
|
+
const runScanChecks = async ({ targetUrl, html, chunks, totalPageSize, requestUrls = [], runtimeSignals }, emit, { onProgress, onPageInfo, ...fetchOptions }) => {
|
|
28
|
+
const origin = new URL(targetUrl).origin;
|
|
29
|
+
onProgress?.(20, "Analyzing html attributes and hreflang tags...");
|
|
30
|
+
const langTag = require_scan_parseHtml.extractHtmlLang(html);
|
|
31
|
+
const hreflangs = require_scan_parseHtml.extractHreflangs(html);
|
|
32
|
+
const routing = require_scan_detection_detectRoutingStrategy.detectRoutingStrategy({
|
|
33
|
+
pageUrl: targetUrl,
|
|
34
|
+
htmlLang: langTag,
|
|
35
|
+
hreflangs
|
|
36
|
+
});
|
|
37
|
+
const signals = {
|
|
38
|
+
targetUrl,
|
|
39
|
+
langTag,
|
|
40
|
+
dirTag: require_scan_parseHtml.extractHtmlDir(html),
|
|
41
|
+
ogLocale: require_scan_parseHtml.extractOgLocale(html),
|
|
42
|
+
ogLocaleAlternates: require_scan_parseHtml.extractOgLocaleAlternates(html),
|
|
43
|
+
canonicalHref: require_scan_parseHtml.extractCanonicalHref(html),
|
|
44
|
+
hreflangs,
|
|
45
|
+
anchors: require_scan_parseHtml.extractAnchors(html),
|
|
46
|
+
routing,
|
|
47
|
+
pageLocale: routing.urlLocale ?? langTag
|
|
48
|
+
};
|
|
49
|
+
const locales = [.../* @__PURE__ */ new Set([...langTag ? [langTag] : [], ...routing.locales])];
|
|
50
|
+
const pageInfo = {
|
|
51
|
+
metadata: extractPageMetadata(html, targetUrl),
|
|
52
|
+
routing,
|
|
53
|
+
locales,
|
|
54
|
+
technologies: require_scan_detection_detectTechnologies.detectTechnologies({
|
|
55
|
+
pageUrl: targetUrl,
|
|
56
|
+
html,
|
|
57
|
+
resourceUrls: [.../* @__PURE__ */ new Set([...require_scan_parseHtml.extractResourceUrls(html, targetUrl), ...requestUrls])],
|
|
58
|
+
scripts: chunks.map(({ content }) => content),
|
|
59
|
+
...runtimeSignals
|
|
60
|
+
})
|
|
61
|
+
};
|
|
62
|
+
onPageInfo?.(pageInfo);
|
|
63
|
+
require_scan_checks.checkHtmlLang(signals, emit);
|
|
64
|
+
require_scan_checks.checkHtmlDir(signals, emit);
|
|
65
|
+
require_scan_checks.checkLocaleConsistency(signals, emit);
|
|
66
|
+
require_scan_checks.checkOgLocale(signals, emit);
|
|
67
|
+
require_scan_checks.checkCanonical(signals, emit);
|
|
68
|
+
require_scan_checks.checkHreflang(signals, emit);
|
|
69
|
+
require_scan_checks.checkXDefault(signals, emit);
|
|
70
|
+
onProgress?.(35, "Checking hreflang alternates...");
|
|
71
|
+
await require_scan_checks.checkHreflangReciprocity(signals, fetchOptions, emit);
|
|
72
|
+
onProgress?.(50, "Analysing bundle content & leakage...");
|
|
73
|
+
const bundle = require_scan_checks.checkBundleContent(chunks, html, signals.pageLocale, targetUrl, totalPageSize, emit);
|
|
74
|
+
onProgress?.(60, "Analyzing internal links...");
|
|
75
|
+
require_scan_checks.checkInternalLinks(signals, emit);
|
|
76
|
+
onProgress?.(70, "Checking robots.txt...");
|
|
77
|
+
const localizedUrls = hreflangs.flatMap(({ href }) => require_scan_detection_url.parseUrl(href, targetUrl)?.href ?? []);
|
|
78
|
+
const declaredSitemapUrls = await require_scan_checks.checkRobots(origin, localizedUrls, fetchOptions, emit);
|
|
79
|
+
onProgress?.(85, "Checking sitemaps...");
|
|
80
|
+
await require_scan_checks.checkSitemap(origin, declaredSitemapUrls, routing, fetchOptions, emit);
|
|
81
|
+
return {
|
|
82
|
+
...pageInfo,
|
|
83
|
+
bundle
|
|
84
|
+
};
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
//#endregion
|
|
88
|
+
exports.runScanChecks = runScanChecks;
|
|
89
|
+
//# sourceMappingURL=runScanChecks.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runScanChecks.cjs","names":["extractOgImage","extractTitle","extractMetaDescription","parseUrl","extractHtmlLang","extractHreflangs","detectRoutingStrategy","extractHtmlDir","extractOgLocale","extractOgLocaleAlternates","extractCanonicalHref","extractAnchors","detectTechnologies","extractResourceUrls","checkHreflangReciprocity","checkBundleContent","checkRobots","checkSitemap"],"sources":["../../../src/scan/runScanChecks.ts"],"sourcesContent":["import {\n checkBundleContent,\n checkCanonical,\n checkHreflang,\n checkHreflangReciprocity,\n checkHtmlDir,\n checkHtmlLang,\n checkInternalLinks,\n checkLocaleConsistency,\n checkOgLocale,\n checkRobots,\n checkSitemap,\n checkXDefault,\n type EmitScanEvent,\n type PageSignals,\n} from './checks';\nimport { detectRoutingStrategy } from './detection/detectRoutingStrategy';\nimport {\n detectTechnologies,\n type TechnologyDetectionInput,\n} from './detection/detectTechnologies';\nimport { parseUrl } from './detection/url';\nimport type { ScanFetchOptions } from './fetchText';\nimport {\n extractAnchors,\n extractCanonicalHref,\n extractHreflangs,\n extractHtmlDir,\n extractHtmlLang,\n extractMetaDescription,\n extractOgImage,\n extractOgLocale,\n extractOgLocaleAlternates,\n extractResourceUrls,\n extractTitle,\n} from './parseHtml';\nimport type { BundleChunkInput, PageMetadata, ScanResult } from './types';\n\n/** Page material gathered by the caller (fetch, puppeteer, browser…). */\nexport type ScanPageInput = {\n /** Absolute URL of the scanned page. */\n targetUrl: string;\n /** HTML of the page, rendered when possible. */\n html: string;\n /** Fetched JavaScript chunks. */\n chunks: BundleChunkInput[];\n /** Total transferred bytes measured for the page. */\n totalPageSize: number;\n /** Network request URLs observed while loading the page. */\n requestUrls?: string[];\n /** Live-page signals (globals, storage keys) for technology detection. */\n runtimeSignals?: Pick<\n TechnologyDetectionInput,\n 'globals' | 'storageKeys' | 'globalVersions'\n >;\n};\n\n/** Page-level information available before the network-bound checks. */\nexport type ScanPageInfo = Pick<\n ScanResult,\n 'metadata' | 'routing' | 'technologies' | 'locales'\n>;\n\n/** Options of {@link runScanChecks}. */\nexport type RunScanChecksOptions = ScanFetchOptions & {\n /** Called before each step, e.g. to stream a progress bar. */\n onProgress?: (progress: number, message: string) => void;\n /** Called once the page itself is analyzed (metadata, routing, stack). */\n onPageInfo?: (pageInfo: ScanPageInfo) => void;\n};\n\n/** Output of {@link runScanChecks}. */\nexport type ScanChecksResult = Pick<\n ScanResult,\n 'metadata' | 'routing' | 'technologies' | 'locales' | 'bundle'\n>;\n\n/** Extract title, description and absolute preview image. */\nconst extractPageMetadata = (html: string, targetUrl: string): PageMetadata => {\n const ogImage = extractOgImage(html);\n return {\n title: extractTitle(html),\n description: extractMetaDescription(html),\n image: ogImage ? (parseUrl(ogImage, targetUrl)?.href ?? ogImage) : '',\n };\n};\n\n/**\n * Run every i18n/SEO check on an already loaded page. Single implementation\n * shared by the `intlayer scan` CLI and the hosted audit (`/api/scan`), which\n * only differ in how they load the page.\n *\n * @param input - The loaded page (HTML, chunks, size, runtime signals).\n * @param emit - Receives every check result as soon as it is produced.\n * @param options - Network options and progress callbacks.\n */\nexport const runScanChecks = async (\n {\n targetUrl,\n html,\n chunks,\n totalPageSize,\n requestUrls = [],\n runtimeSignals,\n }: ScanPageInput,\n emit: EmitScanEvent,\n { onProgress, onPageInfo, ...fetchOptions }: RunScanChecksOptions\n): Promise<ScanChecksResult> => {\n const origin = new URL(targetUrl).origin;\n\n onProgress?.(20, 'Analyzing html attributes and hreflang tags...');\n\n const langTag = extractHtmlLang(html);\n const hreflangs = extractHreflangs(html);\n const routing = detectRoutingStrategy({\n pageUrl: targetUrl,\n htmlLang: langTag,\n hreflangs,\n });\n const signals: PageSignals = {\n targetUrl,\n langTag,\n dirTag: extractHtmlDir(html),\n ogLocale: extractOgLocale(html),\n ogLocaleAlternates: extractOgLocaleAlternates(html),\n canonicalHref: extractCanonicalHref(html),\n hreflangs,\n anchors: extractAnchors(html),\n routing,\n pageLocale: routing.urlLocale ?? langTag,\n };\n\n const locales = [\n ...new Set([...(langTag ? [langTag] : []), ...routing.locales]),\n ];\n const pageInfo: ScanPageInfo = {\n metadata: extractPageMetadata(html, targetUrl),\n routing,\n locales,\n technologies: detectTechnologies({\n pageUrl: targetUrl,\n html,\n resourceUrls: [\n ...new Set([...extractResourceUrls(html, targetUrl), ...requestUrls]),\n ],\n scripts: chunks.map(({ content }) => content),\n ...runtimeSignals,\n }),\n };\n onPageInfo?.(pageInfo);\n\n checkHtmlLang(signals, emit);\n checkHtmlDir(signals, emit);\n checkLocaleConsistency(signals, emit);\n checkOgLocale(signals, emit);\n checkCanonical(signals, emit);\n checkHreflang(signals, emit);\n checkXDefault(signals, emit);\n\n onProgress?.(35, 'Checking hreflang alternates...');\n await checkHreflangReciprocity(signals, fetchOptions, emit);\n\n onProgress?.(50, 'Analysing bundle content & leakage...');\n const bundle = checkBundleContent(\n chunks,\n html,\n signals.pageLocale,\n targetUrl,\n totalPageSize,\n emit\n );\n\n onProgress?.(60, 'Analyzing internal links...');\n checkInternalLinks(signals, emit);\n\n onProgress?.(70, 'Checking robots.txt...');\n const localizedUrls = hreflangs.flatMap(\n ({ href }) => parseUrl(href, targetUrl)?.href ?? []\n );\n const declaredSitemapUrls = await checkRobots(\n origin,\n localizedUrls,\n fetchOptions,\n emit\n );\n\n onProgress?.(85, 'Checking sitemaps...');\n await checkSitemap(origin, declaredSitemapUrls, routing, fetchOptions, emit);\n\n return { ...pageInfo, bundle };\n};\n"],"mappings":";;;;;;;;;AA8EA,MAAM,uBAAuB,MAAc,cAAoC;CAC7E,MAAM,UAAUA,sCAAe,IAAI;CACnC,OAAO;EACL,OAAOC,oCAAa,IAAI;EACxB,aAAaC,8CAAuB,IAAI;EACxC,OAAO,UAAWC,oCAAS,SAAS,SAAS,CAAC,EAAE,QAAQ,UAAW;CACrE;AACF;;;;;;;;;;AAWA,MAAa,gBAAgB,OAC3B,EACE,WACA,MACA,QACA,eACA,cAAc,CAAC,GACf,kBAEF,MACA,EAAE,YAAY,YAAY,GAAG,mBACC;CAC9B,MAAM,SAAS,IAAI,IAAI,SAAS,CAAC,CAAC;CAElC,aAAa,IAAI,gDAAgD;CAEjE,MAAM,UAAUC,uCAAgB,IAAI;CACpC,MAAM,YAAYC,wCAAiB,IAAI;CACvC,MAAM,UAAUC,mEAAsB;EACpC,SAAS;EACT,UAAU;EACV;CACF,CAAC;CACD,MAAM,UAAuB;EAC3B;EACA;EACA,QAAQC,sCAAe,IAAI;EAC3B,UAAUC,uCAAgB,IAAI;EAC9B,oBAAoBC,iDAA0B,IAAI;EAClD,eAAeC,4CAAqB,IAAI;EACxC;EACA,SAASC,sCAAe,IAAI;EAC5B;EACA,YAAY,QAAQ,aAAa;CACnC;CAEA,MAAM,UAAU,CACd,mBAAG,IAAI,IAAI,CAAC,GAAI,UAAU,CAAC,OAAO,IAAI,CAAC,GAAI,GAAG,QAAQ,OAAO,CAAC,CAChE;CACA,MAAM,WAAyB;EAC7B,UAAU,oBAAoB,MAAM,SAAS;EAC7C;EACA;EACA,cAAcC,6DAAmB;GAC/B,SAAS;GACT;GACA,cAAc,CACZ,mBAAG,IAAI,IAAI,CAAC,GAAGC,2CAAoB,MAAM,SAAS,GAAG,GAAG,WAAW,CAAC,CACtE;GACA,SAAS,OAAO,KAAK,EAAE,cAAc,OAAO;GAC5C,GAAG;EACL,CAAC;CACH;CACA,aAAa,QAAQ;CAErB,kCAAc,SAAS,IAAI;CAC3B,iCAAa,SAAS,IAAI;CAC1B,2CAAuB,SAAS,IAAI;CACpC,kCAAc,SAAS,IAAI;CAC3B,mCAAe,SAAS,IAAI;CAC5B,kCAAc,SAAS,IAAI;CAC3B,kCAAc,SAAS,IAAI;CAE3B,aAAa,IAAI,iCAAiC;CAClD,MAAMC,6CAAyB,SAAS,cAAc,IAAI;CAE1D,aAAa,IAAI,uCAAuC;CACxD,MAAM,SAASC,uCACb,QACA,MACA,QAAQ,YACR,WACA,eACA,IACF;CAEA,aAAa,IAAI,6BAA6B;CAC9C,uCAAmB,SAAS,IAAI;CAEhC,aAAa,IAAI,wBAAwB;CACzC,MAAM,gBAAgB,UAAU,SAC7B,EAAE,WAAWZ,oCAAS,MAAM,SAAS,CAAC,EAAE,QAAQ,CAAC,CACpD;CACA,MAAM,sBAAsB,MAAMa,gCAChC,QACA,eACA,cACA,IACF;CAEA,aAAa,IAAI,sBAAsB;CACvC,MAAMC,iCAAa,QAAQ,qBAAqB,SAAS,cAAc,IAAI;CAE3E,OAAO;EAAE,GAAG;EAAU;CAAO;AAC/B"}
|