@rivi-software/agentic-seo-check 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +124 -18
- package/dist/brand.d.ts +11 -0
- package/dist/brand.js +22 -0
- package/dist/brand.js.map +1 -0
- package/dist/cli.js +174 -86
- package/dist/cli.js.map +1 -1
- package/dist/core/aeo/audit.d.ts +44 -0
- package/dist/core/aeo/audit.js +310 -0
- package/dist/core/aeo/audit.js.map +1 -0
- package/dist/core/aeo/bots.d.ts +30 -0
- package/dist/core/aeo/bots.js +299 -0
- package/dist/core/aeo/bots.js.map +1 -0
- package/dist/core/aeo/llms-txt.d.ts +8 -0
- package/dist/core/aeo/llms-txt.js +88 -0
- package/dist/core/aeo/llms-txt.js.map +1 -0
- package/dist/core/aeo/page.d.ts +53 -0
- package/dist/core/aeo/page.js +312 -0
- package/dist/core/aeo/page.js.map +1 -0
- package/dist/core/aeo/probe.d.ts +10 -0
- package/dist/core/aeo/probe.js +53 -0
- package/dist/core/aeo/probe.js.map +1 -0
- package/dist/core/aeo/site-rules.d.ts +13 -0
- package/dist/core/aeo/site-rules.js +178 -0
- package/dist/core/aeo/site-rules.js.map +1 -0
- package/dist/core/audit.d.ts +37 -16
- package/dist/core/audit.js +389 -249
- package/dist/core/audit.js.map +1 -1
- package/dist/core/concurrency.d.ts +4 -4
- package/dist/core/concurrency.js +33 -23
- package/dist/core/concurrency.js.map +1 -1
- package/dist/core/crawl.d.ts +9 -5
- package/dist/core/crawl.js +271 -123
- package/dist/core/crawl.js.map +1 -1
- package/dist/core/fetch.d.ts +5 -5
- package/dist/core/fetch.js +151 -100
- package/dist/core/fetch.js.map +1 -1
- package/dist/core/html/answer.d.ts +4 -0
- package/dist/core/html/answer.js +198 -0
- package/dist/core/html/answer.js.map +1 -0
- package/dist/core/html/dom.d.ts +17 -0
- package/dist/core/html/dom.js +83 -0
- package/dist/core/html/dom.js.map +1 -0
- package/dist/core/html/extract.d.ts +2 -2
- package/dist/core/html/extract.js +102 -111
- package/dist/core/html/extract.js.map +1 -1
- package/dist/core/html/structured-data.d.ts +16 -6
- package/dist/core/html/structured-data.js +180 -145
- package/dist/core/html/structured-data.js.map +1 -1
- package/dist/core/html/text.d.ts +14 -14
- package/dist/core/html/text.js +110 -107
- package/dist/core/html/text.js.map +1 -1
- package/dist/core/lighthouse.d.ts +26 -25
- package/dist/core/lighthouse.js +181 -105
- package/dist/core/lighthouse.js.map +1 -1
- package/dist/core/performance-rules.d.ts +2 -2
- package/dist/core/performance-rules.js +105 -33
- package/dist/core/performance-rules.js.map +1 -1
- package/dist/core/render-rules.d.ts +5 -4
- package/dist/core/render-rules.js +129 -62
- package/dist/core/render-rules.js.map +1 -1
- package/dist/core/render.d.ts +2 -2
- package/dist/core/render.js +29 -23
- package/dist/core/render.js.map +1 -1
- package/dist/core/report/markdown.d.ts +4 -2
- package/dist/core/report/markdown.js +210 -129
- package/dist/core/report/markdown.js.map +1 -1
- package/dist/core/robots.d.ts +12 -5
- package/dist/core/robots.js +43 -17
- package/dist/core/robots.js.map +1 -1
- package/dist/core/rules/aeo.d.ts +11 -0
- package/dist/core/rules/aeo.js +423 -0
- package/dist/core/rules/aeo.js.map +1 -0
- package/dist/core/rules/canonical.d.ts +1 -1
- package/dist/core/rules/canonical.js +93 -57
- package/dist/core/rules/canonical.js.map +1 -1
- package/dist/core/rules/content.js +40 -26
- package/dist/core/rules/content.js.map +1 -1
- package/dist/core/rules/document.js +64 -41
- package/dist/core/rules/document.js.map +1 -1
- package/dist/core/rules/headings.js +58 -38
- package/dist/core/rules/headings.js.map +1 -1
- package/dist/core/rules/hreflang.js +70 -38
- package/dist/core/rules/hreflang.js.map +1 -1
- package/dist/core/rules/images.js +44 -26
- package/dist/core/rules/images.js.map +1 -1
- package/dist/core/rules/index.d.ts +10 -6
- package/dist/core/rules/index.js +43 -36
- package/dist/core/rules/index.js.map +1 -1
- package/dist/core/rules/indexing.d.ts +2 -2
- package/dist/core/rules/indexing.js +48 -26
- package/dist/core/rules/indexing.js.map +1 -1
- package/dist/core/rules/keyword.js +84 -47
- package/dist/core/rules/keyword.js.map +1 -1
- package/dist/core/rules/links.js +72 -37
- package/dist/core/rules/links.js.map +1 -1
- package/dist/core/rules/meta.js +55 -34
- package/dist/core/rules/meta.js.map +1 -1
- package/dist/core/rules/response.js +75 -41
- package/dist/core/rules/response.js.map +1 -1
- package/dist/core/rules/rule.d.ts +6 -6
- package/dist/core/rules/rule.js +36 -36
- package/dist/core/rules/rule.js.map +1 -1
- package/dist/core/rules/social.js +86 -45
- package/dist/core/rules/social.js.map +1 -1
- package/dist/core/rules/structured-data.d.ts +6 -4
- package/dist/core/rules/structured-data.js +314 -102
- package/dist/core/rules/structured-data.js.map +1 -1
- package/dist/core/rules/title.js +48 -33
- package/dist/core/rules/title.js.map +1 -1
- package/dist/core/score.d.ts +5 -3
- package/dist/core/score.js +11 -9
- package/dist/core/score.js.map +1 -1
- package/dist/core/site-rules.d.ts +3 -3
- package/dist/core/site-rules.js +409 -198
- package/dist/core/site-rules.js.map +1 -1
- package/dist/core/sitemap.d.ts +5 -5
- package/dist/core/sitemap.js +109 -64
- package/dist/core/sitemap.js.map +1 -1
- package/dist/core/static.d.ts +2 -2
- package/dist/core/static.js +210 -112
- package/dist/core/static.js.map +1 -1
- package/dist/core/summary.d.ts +2 -2
- package/dist/core/summary.js +11 -6
- package/dist/core/summary.js.map +1 -1
- package/dist/core/types.d.ts +189 -14
- package/dist/core/types.js +15 -8
- package/dist/core/types.js.map +1 -1
- package/dist/core/url.d.ts +11 -11
- package/dist/core/url.js +122 -70
- package/dist/core/url.js.map +1 -1
- package/dist/index.d.ts +15 -11
- package/dist/index.js +11 -9
- package/dist/index.js.map +1 -1
- package/dist/mcp-bin.js +7 -4
- package/dist/mcp-bin.js.map +1 -1
- package/dist/mcp.d.ts +3 -3
- package/dist/mcp.js +437 -121
- package/dist/mcp.js.map +1 -1
- package/dist/version.d.ts +0 -1
- package/dist/version.js +4 -2
- package/dist/version.js.map +1 -1
- package/package.json +32 -10
- package/skills/seo-audit/SKILL.md +8 -2
package/dist/core/crawl.js
CHANGED
|
@@ -1,23 +1,122 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { DEFAULT_USER_AGENT, fetchPage, isHtmlResponse, probeUrl, toResponseInfo } from
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { isLocalHost } from
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
|
|
1
|
+
import { mapConcurrent, sleep } from "./concurrency.js";
|
|
2
|
+
import { DEFAULT_USER_AGENT, fetchPage, isHtmlResponse, probeUrl, toResponseInfo, } from "./fetch.js";
|
|
3
|
+
import { extractPage } from "./html/extract.js";
|
|
4
|
+
import { loadRobotsTxt } from "./robots.js";
|
|
5
|
+
import { isLocalHost } from "./rules/canonical.js";
|
|
6
|
+
import { loadSitemaps } from "./sitemap.js";
|
|
7
|
+
import { hostOf, isSameSite, looksLikeHtml, normalizeUrl, stripWww, } from "./url.js";
|
|
8
|
+
/**
|
|
9
|
+
* Works through a queue that grows while it runs: each crawled page can add the links it found.
|
|
10
|
+
* At most `concurrency` items run at once, and with `delayMs` each slot waits that long after its
|
|
11
|
+
* item before it takes the next one. An item that throws is left out; the rest go on.
|
|
12
|
+
*/
|
|
13
|
+
const runQueue = async (queue, concurrency, delayMs, work) => {
|
|
14
|
+
const limit = Math.max(1, concurrency);
|
|
15
|
+
const running = new Set();
|
|
16
|
+
const launch = (item) => {
|
|
17
|
+
const task = (async () => {
|
|
18
|
+
try {
|
|
19
|
+
await work(item);
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
// The page is left out of the crawl; the rest of the queue goes on.
|
|
23
|
+
}
|
|
24
|
+
if (delayMs > 0) {
|
|
25
|
+
await sleep(delayMs);
|
|
26
|
+
}
|
|
27
|
+
})().finally(() => running.delete(task));
|
|
28
|
+
running.add(task);
|
|
29
|
+
};
|
|
30
|
+
for (;;) {
|
|
31
|
+
while (running.size < limit) {
|
|
32
|
+
const item = queue.shift();
|
|
33
|
+
if (!item) {
|
|
34
|
+
break;
|
|
35
|
+
}
|
|
36
|
+
launch(item);
|
|
37
|
+
}
|
|
38
|
+
if (running.size === 0) {
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
// oxlint-disable-next-line eslint/no-await-in-loop -- the queue grows as pages finish, so what to start next is only known once a slot frees
|
|
42
|
+
await Promise.race(running);
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
/** Adds `source` to the set of pages that link to `target`. */
|
|
46
|
+
const addSource = (sources, target, source) => {
|
|
47
|
+
let set = sources.get(target);
|
|
48
|
+
if (!set) {
|
|
49
|
+
set = new Set();
|
|
50
|
+
sources.set(target, set);
|
|
51
|
+
}
|
|
52
|
+
set.add(source);
|
|
53
|
+
};
|
|
54
|
+
const runProbes = async (startUrl, fetchOpts) => {
|
|
55
|
+
const probes = {};
|
|
56
|
+
const url = new URL(startUrl);
|
|
57
|
+
const soft404Url = new URL(`/agentic-seo-probe-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`, url.origin).href;
|
|
58
|
+
const soft = await probeUrl(soft404Url, fetchOpts);
|
|
59
|
+
probes.soft404 = soft.error ? null : { url: soft404Url, status: soft.status };
|
|
60
|
+
if (isLocalHost(startUrl)) {
|
|
61
|
+
return probes;
|
|
62
|
+
}
|
|
63
|
+
if (url.protocol === "https:") {
|
|
64
|
+
const httpUrl = `http://${url.host}/`;
|
|
65
|
+
const res = await probeUrl(httpUrl, { ...fetchOpts, timeoutMs: 10_000 });
|
|
66
|
+
probes.httpRedirect = res.error
|
|
67
|
+
? null
|
|
68
|
+
: {
|
|
69
|
+
url: httpUrl,
|
|
70
|
+
status: res.status,
|
|
71
|
+
finalUrl: res.finalUrl,
|
|
72
|
+
redirectsToHttps: res.finalUrl.startsWith("https://"),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
const host = url.hostname;
|
|
76
|
+
const variantHost = host.startsWith("www.") ? stripWww(host) : `www.${host}`;
|
|
77
|
+
const variantUrl = `${url.protocol}//${variantHost}/`;
|
|
78
|
+
const res = await probeUrl(variantUrl, { ...fetchOpts, timeoutMs: 10_000 });
|
|
79
|
+
probes.hostVariant = res.error
|
|
80
|
+
? null
|
|
81
|
+
: {
|
|
82
|
+
url: variantUrl,
|
|
83
|
+
status: res.status,
|
|
84
|
+
finalUrl: res.finalUrl,
|
|
85
|
+
redirectsToMain: (hostOf(res.finalUrl) ?? "") === host,
|
|
86
|
+
};
|
|
87
|
+
return probes;
|
|
88
|
+
};
|
|
89
|
+
/** The progress callback of a caller that does not listen for progress. */
|
|
90
|
+
const ignoreProgress = () => {
|
|
91
|
+
// Nobody to tell.
|
|
92
|
+
};
|
|
93
|
+
export const crawlSite = async (startInput, options) => {
|
|
9
94
|
const started = performance.now();
|
|
10
|
-
const progress = options.onProgress ??
|
|
95
|
+
const progress = options.onProgress ?? ignoreProgress;
|
|
11
96
|
const startUrl = normalizeUrl(startInput) ?? startInput;
|
|
12
|
-
const origin = new URL(startUrl)
|
|
13
|
-
const startHost = hostOf(startUrl) ??
|
|
97
|
+
const { origin } = new URL(startUrl);
|
|
98
|
+
const startHost = hostOf(startUrl) ?? "";
|
|
14
99
|
const userAgent = options.fetch.userAgent ?? DEFAULT_USER_AGENT;
|
|
15
|
-
progress({ phase:
|
|
100
|
+
progress({ phase: "robots", message: `Fetching ${origin}/robots.txt` });
|
|
16
101
|
const robots = await loadRobotsTxt(origin, userAgent, options.fetch);
|
|
17
102
|
const allows = options.respectRobots ? robots.allows : () => true;
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
103
|
+
const siteChecks = options.siteChecks ?? true;
|
|
104
|
+
let sitemaps = {
|
|
105
|
+
sources: [],
|
|
106
|
+
urls: [],
|
|
107
|
+
urlCount: 0,
|
|
108
|
+
errors: [],
|
|
109
|
+
};
|
|
110
|
+
if (siteChecks) {
|
|
111
|
+
progress({ phase: "sitemap", message: "Fetching sitemaps" });
|
|
112
|
+
const sitemapCandidates = [
|
|
113
|
+
...new Set([
|
|
114
|
+
...robots.info.sitemaps,
|
|
115
|
+
new URL("/sitemap.xml", origin).href,
|
|
116
|
+
]),
|
|
117
|
+
];
|
|
118
|
+
sitemaps = await loadSitemaps(sitemapCandidates, options.fetch);
|
|
119
|
+
}
|
|
21
120
|
const visited = new Set([startUrl]);
|
|
22
121
|
const discovered = new Set([startUrl]);
|
|
23
122
|
const linkChecks = new Map();
|
|
@@ -27,103 +126,194 @@ export async function crawlSite(startInput, options) {
|
|
|
27
126
|
const blocked = [];
|
|
28
127
|
let limitSkips = 0;
|
|
29
128
|
const accepts = (url) => {
|
|
30
|
-
if (options.includePatterns.length &&
|
|
129
|
+
if (options.includePatterns.length &&
|
|
130
|
+
!options.includePatterns.some((re) => re.test(url))) {
|
|
31
131
|
return false;
|
|
32
|
-
|
|
132
|
+
}
|
|
133
|
+
if (options.excludePatterns.some((re) => re.test(url))) {
|
|
33
134
|
return false;
|
|
135
|
+
}
|
|
34
136
|
return true;
|
|
35
137
|
};
|
|
36
138
|
const queue = [{ url: startUrl, depth: 0 }];
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
const res = await fetchPage(item.url, options.fetch);
|
|
43
|
-
const requestedKey = item.url;
|
|
44
|
-
const finalKey = normalizeUrl(res.finalUrl) ?? res.finalUrl;
|
|
45
|
-
linkChecks.set(requestedKey, { url: item.url, status: res.status, finalUrl: res.finalUrl, redirectChain: res.redirectChain, error: res.error });
|
|
46
|
-
if (finalKey !== requestedKey) {
|
|
47
|
-
if (!linkChecks.has(finalKey))
|
|
48
|
-
linkChecks.set(finalKey, { url: res.finalUrl, status: res.status, finalUrl: res.finalUrl, redirectChain: [], error: res.error });
|
|
49
|
-
if (visited.has(finalKey) && records.some((r) => r.url === finalKey))
|
|
50
|
-
return;
|
|
51
|
-
visited.add(finalKey);
|
|
52
|
-
if (!isSameSite(finalKey, startHost, options.includeSubdomains))
|
|
53
|
-
return; // redirected off-site
|
|
54
|
-
}
|
|
55
|
-
progress({ phase: 'crawl', message: `${res.status ?? 'ERR'} ${res.finalUrl}`, crawled: records.length + 1, discovered: discovered.size, url: res.finalUrl });
|
|
56
|
-
if (res.error || typeof res.body !== 'string' || !isHtmlResponse(res)) {
|
|
57
|
-
if (res.status !== null && res.status < 400 && !res.error && typeof res.body === 'string' && !isHtmlResponse(res))
|
|
58
|
-
return; // non-HTML asset
|
|
59
|
-
// Record error pages so page-level findings (response.error-status) appear for the start URL.
|
|
60
|
-
if (item.depth === 0 || res.status === null) {
|
|
61
|
-
const page = extractPage(res.body ?? '', { url: res.finalUrl, siteHost: startHost, includeSubdomains: options.includeSubdomains });
|
|
62
|
-
records.push({ url: finalKey, requestedUrl: item.url, page, response: toResponseInfo(res), depth: item.depth });
|
|
63
|
-
}
|
|
64
|
-
return;
|
|
65
|
-
}
|
|
66
|
-
const page = extractPage(res.body, { url: res.finalUrl, siteHost: startHost, includeSubdomains: options.includeSubdomains });
|
|
67
|
-
records.push({ url: finalKey, requestedUrl: item.url, page, response: toResponseInfo(res), depth: item.depth });
|
|
68
|
-
for (const link of page.links) {
|
|
69
|
-
if (!link.url)
|
|
70
|
-
continue;
|
|
71
|
-
const target = normalizeUrl(link.url);
|
|
72
|
-
if (!target)
|
|
139
|
+
/** Queues the internal links a crawled page has, and remembers every link's sources. */
|
|
140
|
+
const followLinks = (links, from, depth) => {
|
|
141
|
+
for (const link of links) {
|
|
142
|
+
const target = link.url ? normalizeUrl(link.url) : null;
|
|
143
|
+
if (!target) {
|
|
73
144
|
continue;
|
|
145
|
+
}
|
|
74
146
|
if (!link.internal) {
|
|
75
|
-
|
|
76
|
-
externalSources.set(target, new Set());
|
|
77
|
-
externalSources.get(target).add(finalKey);
|
|
147
|
+
addSource(externalSources, target, from);
|
|
78
148
|
continue;
|
|
79
149
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
continue;
|
|
85
|
-
if (discovered.has(target))
|
|
150
|
+
addSource(linkSources, target, from);
|
|
151
|
+
if (!looksLikeHtml(target) ||
|
|
152
|
+
!accepts(target) ||
|
|
153
|
+
discovered.has(target)) {
|
|
86
154
|
continue;
|
|
155
|
+
}
|
|
87
156
|
discovered.add(target);
|
|
88
|
-
if (
|
|
157
|
+
if (depth + 1 > options.maxDepth) {
|
|
89
158
|
continue;
|
|
159
|
+
}
|
|
90
160
|
if (visited.size >= options.maxPages) {
|
|
91
|
-
limitSkips
|
|
161
|
+
limitSkips += 1;
|
|
92
162
|
continue;
|
|
93
163
|
}
|
|
94
164
|
visited.add(target);
|
|
95
|
-
queue.push({ url: target, depth:
|
|
165
|
+
queue.push({ url: target, depth: depth + 1 });
|
|
96
166
|
}
|
|
97
167
|
};
|
|
168
|
+
/**
|
|
169
|
+
* Records where a redirected request ended. False when there is nothing more to crawl there:
|
|
170
|
+
* the redirect led to a page crawled already, or off the site.
|
|
171
|
+
*/
|
|
172
|
+
const keepsRedirectTarget = (res, finalKey) => {
|
|
173
|
+
if (!linkChecks.has(finalKey)) {
|
|
174
|
+
linkChecks.set(finalKey, {
|
|
175
|
+
url: res.finalUrl,
|
|
176
|
+
status: res.status,
|
|
177
|
+
finalUrl: res.finalUrl,
|
|
178
|
+
redirectChain: [],
|
|
179
|
+
error: res.error,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
if (visited.has(finalKey) && records.some((r) => r.url === finalKey)) {
|
|
183
|
+
return false;
|
|
184
|
+
}
|
|
185
|
+
visited.add(finalKey);
|
|
186
|
+
return isSameSite(finalKey, startHost, options.includeSubdomains);
|
|
187
|
+
};
|
|
188
|
+
/** A response that is no HTML page: an asset is skipped, an error page may still be recorded. */
|
|
189
|
+
const recordUnreadable = (item, res, finalKey) => {
|
|
190
|
+
const isAsset = res.status !== null &&
|
|
191
|
+
res.status < 400 &&
|
|
192
|
+
!res.error &&
|
|
193
|
+
res.body !== null &&
|
|
194
|
+
!isHtmlResponse(res);
|
|
195
|
+
// Record error pages so page-level findings (response.error-status) appear for the start URL.
|
|
196
|
+
if (isAsset || (item.depth !== 0 && res.status !== null)) {
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
const page = extractPage(res.body ?? "", {
|
|
200
|
+
url: res.finalUrl,
|
|
201
|
+
siteHost: startHost,
|
|
202
|
+
includeSubdomains: options.includeSubdomains,
|
|
203
|
+
});
|
|
204
|
+
records.push({
|
|
205
|
+
url: finalKey,
|
|
206
|
+
requestedUrl: item.url,
|
|
207
|
+
page,
|
|
208
|
+
response: toResponseInfo(res),
|
|
209
|
+
depth: item.depth,
|
|
210
|
+
});
|
|
211
|
+
};
|
|
212
|
+
const processItem = async (item) => {
|
|
213
|
+
if (!allows(item.url)) {
|
|
214
|
+
blocked.push(item.url);
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
const res = await fetchPage(item.url, options.fetch);
|
|
218
|
+
const requestedKey = item.url;
|
|
219
|
+
const finalKey = normalizeUrl(res.finalUrl) ?? res.finalUrl;
|
|
220
|
+
linkChecks.set(requestedKey, {
|
|
221
|
+
url: item.url,
|
|
222
|
+
status: res.status,
|
|
223
|
+
finalUrl: res.finalUrl,
|
|
224
|
+
redirectChain: res.redirectChain,
|
|
225
|
+
error: res.error,
|
|
226
|
+
});
|
|
227
|
+
if (finalKey !== requestedKey && !keepsRedirectTarget(res, finalKey)) {
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
progress({
|
|
231
|
+
phase: "crawl",
|
|
232
|
+
message: `${res.status ?? "ERR"} ${res.finalUrl}`,
|
|
233
|
+
crawled: records.length + 1,
|
|
234
|
+
discovered: discovered.size,
|
|
235
|
+
url: res.finalUrl,
|
|
236
|
+
});
|
|
237
|
+
if (res.error || res.body === null || !isHtmlResponse(res)) {
|
|
238
|
+
recordUnreadable(item, res, finalKey);
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
const page = extractPage(res.body, {
|
|
242
|
+
url: res.finalUrl,
|
|
243
|
+
siteHost: startHost,
|
|
244
|
+
includeSubdomains: options.includeSubdomains,
|
|
245
|
+
});
|
|
246
|
+
records.push({
|
|
247
|
+
url: finalKey,
|
|
248
|
+
requestedUrl: item.url,
|
|
249
|
+
page,
|
|
250
|
+
response: toResponseInfo(res),
|
|
251
|
+
depth: item.depth,
|
|
252
|
+
});
|
|
253
|
+
followLinks(page.links, finalKey, item.depth);
|
|
254
|
+
};
|
|
98
255
|
await runQueue(queue, options.concurrency, options.delayMs, processItem);
|
|
99
256
|
/* Check internal link targets that were discovered but not crawled (limits, depth, non-HTML). */
|
|
100
|
-
progress({ phase:
|
|
101
|
-
const toCheck = [...linkSources.keys()]
|
|
257
|
+
progress({ phase: "links", message: "Checking internal link targets" });
|
|
258
|
+
const toCheck = [...linkSources.keys()]
|
|
259
|
+
.filter((u) => !linkChecks.has(u) && allows(u))
|
|
260
|
+
.slice(0, options.maxLinkChecks);
|
|
102
261
|
const sitemapExtra = sitemaps.urls
|
|
103
262
|
.map((u) => normalizeUrl(u) ?? u)
|
|
104
|
-
.filter((u) => isSameSite(u, startHost, options.includeSubdomains) &&
|
|
263
|
+
.filter((u) => isSameSite(u, startHost, options.includeSubdomains) &&
|
|
264
|
+
!linkChecks.has(u) &&
|
|
265
|
+
!toCheck.includes(u) &&
|
|
266
|
+
allows(u))
|
|
105
267
|
.slice(0, Math.max(0, options.maxLinkChecks - toCheck.length));
|
|
106
|
-
for (const
|
|
107
|
-
if (!blocked.includes(
|
|
108
|
-
blocked.push(
|
|
268
|
+
for (const target of [...linkSources.keys()].filter((u) => !allows(u))) {
|
|
269
|
+
if (!blocked.includes(target)) {
|
|
270
|
+
blocked.push(target);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
109
273
|
await mapConcurrent([...toCheck, ...sitemapExtra], options.concurrency, async (u) => {
|
|
110
274
|
const res = await probeUrl(u, options.fetch);
|
|
111
|
-
linkChecks.set(u, {
|
|
112
|
-
|
|
275
|
+
linkChecks.set(u, {
|
|
276
|
+
url: u,
|
|
277
|
+
status: res.status,
|
|
278
|
+
finalUrl: res.finalUrl,
|
|
279
|
+
redirectChain: res.redirectChain,
|
|
280
|
+
error: res.error,
|
|
281
|
+
});
|
|
282
|
+
if (options.delayMs) {
|
|
113
283
|
await sleep(options.delayMs);
|
|
284
|
+
}
|
|
114
285
|
});
|
|
115
286
|
let externalChecks;
|
|
116
287
|
if (options.checkExternalLinks) {
|
|
117
|
-
progress({
|
|
118
|
-
|
|
288
|
+
progress({
|
|
289
|
+
phase: "links",
|
|
290
|
+
message: `Checking ${Math.min(externalSources.size, options.maxLinkChecks)} external links`,
|
|
291
|
+
});
|
|
292
|
+
const checks = new Map();
|
|
293
|
+
externalChecks = checks;
|
|
119
294
|
const targets = [...externalSources.keys()].slice(0, options.maxLinkChecks);
|
|
120
295
|
await mapConcurrent(targets, Math.min(options.concurrency, 4), async (u) => {
|
|
121
|
-
const res = await probeUrl(u, {
|
|
122
|
-
|
|
296
|
+
const res = await probeUrl(u, {
|
|
297
|
+
...options.fetch,
|
|
298
|
+
timeoutMs: Math.min(options.fetch.timeoutMs ?? 15_000, 10_000),
|
|
299
|
+
});
|
|
300
|
+
checks.set(u, {
|
|
301
|
+
url: u,
|
|
302
|
+
status: res.status,
|
|
303
|
+
finalUrl: res.finalUrl,
|
|
304
|
+
redirectChain: res.redirectChain,
|
|
305
|
+
error: res.error,
|
|
306
|
+
});
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
let probes = {};
|
|
310
|
+
if (siteChecks) {
|
|
311
|
+
progress({
|
|
312
|
+
phase: "probes",
|
|
313
|
+
message: "Probing soft-404 and host redirects",
|
|
123
314
|
});
|
|
315
|
+
probes = await runProbes(startUrl, options.fetch);
|
|
124
316
|
}
|
|
125
|
-
progress({ phase: 'probes', message: 'Probing soft-404 and host redirects' });
|
|
126
|
-
const probes = await runProbes(startUrl, options.fetch);
|
|
127
317
|
const stats = {
|
|
128
318
|
pagesCrawled: records.length,
|
|
129
319
|
pagesDiscovered: discovered.size,
|
|
@@ -142,54 +332,12 @@ export async function crawlSite(startInput, options) {
|
|
|
142
332
|
externalChecks,
|
|
143
333
|
externalSources: options.checkExternalLinks ? externalSources : undefined,
|
|
144
334
|
robots: robots.info,
|
|
335
|
+
robotsText: robots.text,
|
|
145
336
|
robotsAllows: robots.info.found ? robots.allows : null,
|
|
146
337
|
blockedByRobots: blocked,
|
|
147
338
|
sitemaps,
|
|
148
339
|
probes,
|
|
149
340
|
stats,
|
|
150
341
|
};
|
|
151
|
-
}
|
|
152
|
-
async function runQueue(queue, concurrency, delayMs, work) {
|
|
153
|
-
let active = 0;
|
|
154
|
-
await new Promise((resolve) => {
|
|
155
|
-
const pump = () => {
|
|
156
|
-
while (active < Math.max(1, concurrency) && queue.length) {
|
|
157
|
-
const item = queue.shift();
|
|
158
|
-
active++;
|
|
159
|
-
work(item)
|
|
160
|
-
.catch(() => undefined)
|
|
161
|
-
.finally(() => {
|
|
162
|
-
active--;
|
|
163
|
-
if (delayMs > 0)
|
|
164
|
-
setTimeout(pump, delayMs);
|
|
165
|
-
else
|
|
166
|
-
pump();
|
|
167
|
-
});
|
|
168
|
-
}
|
|
169
|
-
if (active === 0 && queue.length === 0)
|
|
170
|
-
resolve();
|
|
171
|
-
};
|
|
172
|
-
pump();
|
|
173
|
-
});
|
|
174
|
-
}
|
|
175
|
-
async function runProbes(startUrl, fetchOpts) {
|
|
176
|
-
const probes = {};
|
|
177
|
-
const url = new URL(startUrl);
|
|
178
|
-
const soft404Url = new URL(`/agentic-seo-probe-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`, url.origin).href;
|
|
179
|
-
const soft = await probeUrl(soft404Url, fetchOpts);
|
|
180
|
-
probes.soft404 = soft.error ? null : { url: soft404Url, status: soft.status };
|
|
181
|
-
if (isLocalHost(startUrl))
|
|
182
|
-
return probes;
|
|
183
|
-
if (url.protocol === 'https:') {
|
|
184
|
-
const httpUrl = `http://${url.host}/`;
|
|
185
|
-
const res = await probeUrl(httpUrl, { ...fetchOpts, timeoutMs: 10_000 });
|
|
186
|
-
probes.httpRedirect = res.error ? null : { url: httpUrl, status: res.status, finalUrl: res.finalUrl, redirectsToHttps: res.finalUrl.startsWith('https://') };
|
|
187
|
-
}
|
|
188
|
-
const host = url.hostname;
|
|
189
|
-
const variantHost = host.startsWith('www.') ? stripWww(host) : `www.${host}`;
|
|
190
|
-
const variantUrl = `${url.protocol}//${variantHost}/`;
|
|
191
|
-
const res = await probeUrl(variantUrl, { ...fetchOpts, timeoutMs: 10_000 });
|
|
192
|
-
probes.hostVariant = res.error ? null : { url: variantUrl, status: res.status, finalUrl: res.finalUrl, redirectsToMain: (hostOf(res.finalUrl) ?? '') === host };
|
|
193
|
-
return probes;
|
|
194
|
-
}
|
|
342
|
+
};
|
|
195
343
|
//# sourceMappingURL=crawl.js.map
|
package/dist/core/crawl.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,cAAc,EAAqB,MAAM,YAAY,CAAC;AACxH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAGnD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AA8CxD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,UAAkB,EAAE,OAAqB;IACvE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;IACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEhE,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,MAAM,aAAa,EAAE,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;IAElE,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC7D,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxG,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACvD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE;QACvC,IAAI,OAAO,CAAC,eAAe,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACxG,IAAI,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACrE,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEzD,MAAM,WAAW,GAAG,KAAK,EAAE,IAAe,EAAiB,EAAE;QAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO;QACT,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC;QAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;QAC5D,UAAU,CAAC,GAAG,CAAC,YAAY,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAChJ,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAChK,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC;gBAAE,OAAO;YAC7E,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC;gBAAE,OAAO,CAAC,sBAAsB;QACjG,CAAC;QACD,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC7J,IAAI,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YACtE,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;gBAAE,OAAO,CAAC,iBAAiB;YAC5I,8FAA8F;YAC9F,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;gBACnI,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAClH,CAAC;YACD,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC7H,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAEhH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,SAAS;YACxB,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,CAAC,MAAM;gBAAE,SAAS;YACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC;oBAAE,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;gBACzE,eAAe,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YACjE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,SAAS;YACzD,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAS;YACrC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvB,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ;gBAAE,SAAS;YAChD,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrC,UAAU,EAAE,CAAC;gBACb,SAAS;YACX,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEzE,iGAAiG;IACjG,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACvH,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;SAC7H,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACjE,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,aAAa,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,YAAY,CAAC,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAClF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAC9H,IAAI,OAAO,CAAC,OAAO;YAAE,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,IAAI,cAAkD,CAAC;IACvD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC1H,cAAc,GAAG,IAAI,GAAG,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC5E,MAAM,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;YACzE,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;YACpH,cAAe,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QACrI,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAExD,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,eAAe,EAAE,UAAU,CAAC,IAAI;QAChC,YAAY,EAAE,UAAU,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,CAAC;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,YAAY,EAAE,UAAU,GAAG,CAAC;QAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACnD,eAAe,EAAE,OAAO,CAAC,aAAa;KACvC,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,OAAO;QACP,UAAU;QACV,WAAW;QACX,cAAc;QACd,eAAe,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;QACzE,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACtD,eAAe,EAAE,OAAO;QACxB,QAAQ;QACR,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,KAAkB,EAAE,WAAmB,EAAE,OAAe,EAAE,IAAwC;IACxH,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,MAAM,IAAI,GAAG,GAAS,EAAE;YACtB,OAAO,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACzD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;gBAC5B,MAAM,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC;qBACP,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;qBACtB,OAAO,CAAC,GAAG,EAAE;oBACZ,MAAM,EAAE,CAAC;oBACT,IAAI,OAAO,GAAG,CAAC;wBAAE,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;;wBACtC,IAAI,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;YACD,IAAI,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;QACpD,CAAC,CAAC;QACF,IAAI,EAAE,CAAC;IACT,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,SAAuB;IAChE,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;IACvI,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAE9E,IAAI,WAAW,CAAC,QAAQ,CAAC;QAAE,OAAO,MAAM,CAAC;IAEzC,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/J,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,WAAW,GAAG,CAAC;IACtD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,eAAe,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;IAChK,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
1
|
+
{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,kBAAkB,EAClB,SAAS,EACT,cAAc,EACd,QAAQ,EACR,cAAc,GACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAO5C,OAAO,EACL,MAAM,EACN,UAAU,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,GACT,MAAM,UAAU,CAAC;AA4DlB;;;;GAIG;AACH,MAAM,QAAQ,GAAG,KAAK,EACpB,KAAkB,EAClB,WAAmB,EACnB,OAAe,EACf,IAAwC,EACzB,EAAE;IACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IAEzC,MAAM,MAAM,GAAG,CAAC,IAAe,EAAQ,EAAE;QACvC,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE;YACvB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;YAAC,MAAM,CAAC;gBACP,oEAAoE;YACtE,CAAC;YAED,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEzC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC;IAEF,SAAS,CAAC;QACR,OAAO,OAAO,CAAC,IAAI,GAAG,KAAK,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAE3B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM;YACR,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,6IAA6I;QAC7I,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC,CAAC;AAEF,+DAA+D;AAC/D,MAAM,SAAS,GAAG,CAChB,OAAiC,EACjC,MAAc,EACd,MAAc,EACR,EAAE;IACR,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EACrB,QAAgB,EAChB,SAAuB,EACA,EAAE;IACzB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAE9B,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EACzF,GAAG,CAAC,MAAM,CACX,CAAC,IAAI,CAAC;IAEP,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAE9E,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK;YAC7B,CAAC,CAAC,IAAI;YACN,CAAC,CAAC;gBACE,GAAG,EAAE,OAAO;gBACZ,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;aACtD,CAAC;IACR,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,WAAW,GAAG,CAAC;IACtD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,KAAK;QAC5B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC;YACE,GAAG,EAAE,UAAU;YACf,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,eAAe,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI;SACvD,CAAC;IAEN,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,cAAc,GAAG,GAAS,EAAE;IAChC,kBAAkB;AACpB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,UAAkB,EAClB,OAAqB,EACC,EAAE;IACxB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,cAAc,CAAC;IACtD,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;IACxD,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEhE,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,MAAM,aAAa,EAAE,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;IAElE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;IAE9C,IAAI,QAAQ,GAAgB;QAC1B,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,IAAI,UAAU,EAAE,CAAC;QACf,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;QAE7D,MAAM,iBAAiB,GAAG;YACxB,GAAG,IAAI,GAAG,CAAC;gBACT,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ;gBACvB,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI;aACrC,CAAC;SACH,CAAC;QAEF,QAAQ,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAClE,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACvD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE;QACvC,IACE,OAAO,CAAC,eAAe,CAAC,MAAM;YAC9B,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACnD,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEzD,wFAAwF;IACxF,MAAM,WAAW,GAAG,CAClB,KAAiB,EACjB,IAAY,EACZ,KAAa,EACP,EAAE;QACR,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAExD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,SAAS,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;YAErC,IACE,CAAC,aAAa,CAAC,MAAM,CAAC;gBACtB,CAAC,OAAO,CAAC,MAAM,CAAC;gBAChB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EACtB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAEvB,IAAI,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACjC,SAAS;YACX,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrC,UAAU,IAAI,CAAC,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC,CAAC;IAEF;;;OAGG;IACH,MAAM,mBAAmB,GAAG,CAAC,GAAgB,EAAE,QAAgB,EAAW,EAAE;QAC1E,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACvB,GAAG,EAAE,GAAG,CAAC,QAAQ;gBACjB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,EAAE;gBACjB,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtB,OAAO,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpE,CAAC,CAAC;IAEF,iGAAiG;IACjG,MAAM,gBAAgB,GAAG,CACvB,IAAe,EACf,GAAgB,EAChB,QAAgB,EACV,EAAE;QACR,MAAM,OAAO,GACX,GAAG,CAAC,MAAM,KAAK,IAAI;YACnB,GAAG,CAAC,MAAM,GAAG,GAAG;YAChB,CAAC,GAAG,CAAC,KAAK;YACV,GAAG,CAAC,IAAI,KAAK,IAAI;YACjB,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAEvB,8FAA8F;QAC9F,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC;YACzD,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE;YACvC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,KAAK,EAAE,IAAe,EAAiB,EAAE;QAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEvB,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC;QAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;QAE5D,UAAU,CAAC,GAAG,CAAC,YAAY,EAAE;YAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,QAAQ,KAAK,YAAY,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO;QACT,CAAC;QAED,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE;YACjD,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;YAC3B,UAAU,EAAE,UAAU,CAAC,IAAI;YAC3B,GAAG,EAAE,GAAG,CAAC,QAAQ;SAClB,CAAC,CAAC;QAEH,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3D,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;YAEtC,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE;YACjC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;QAEH,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEzE,iGAAiG;IACjG,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IAExE,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;SACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;SAC9C,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAEnC,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAChC,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,UAAU,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC;QACnD,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAClB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpB,MAAM,CAAC,CAAC,CAAC,CACZ;SACA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAEjE,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,aAAa,CACjB,CAAC,GAAG,OAAO,EAAE,GAAG,YAAY,CAAC,EAC7B,OAAO,CAAC,WAAW,EACnB,KAAK,EAAE,CAAC,EAAE,EAAE;QACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE;YAChB,GAAG,EAAE,CAAC;YACN,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CACF,CAAC;IAEF,IAAI,cAAkD,CAAC;IAEvD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,YAAY,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,iBAAiB;SAC5F,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC5C,cAAc,GAAG,MAAM,CAAC;QACxB,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC5E,MAAM,aAAa,CACjB,OAAO,EACP,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAChC,KAAK,EAAE,CAAC,EAAE,EAAE;YACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE;gBAC5B,GAAG,OAAO,CAAC,KAAK;gBAChB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;aAC/D,CAAC,CAAC;YAEH,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE;gBACZ,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC,CACF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,GAAiB,EAAE,CAAC;IAE9B,IAAI,UAAU,EAAE,CAAC;QACf,QAAQ,CAAC;YACP,KAAK,EAAE,QAAQ;YACf,OAAO,EAAE,qCAAqC;SAC/C,CAAC,CAAC;QACH,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,eAAe,EAAE,UAAU,CAAC,IAAI;QAChC,YAAY,EAAE,UAAU,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,CAAC;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,YAAY,EAAE,UAAU,GAAG,CAAC;QAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACnD,eAAe,EAAE,OAAO,CAAC,aAAa;KACvC,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,OAAO;QACP,UAAU;QACV,WAAW;QACX,cAAc;QACd,eAAe,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;QACzE,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,UAAU,EAAE,MAAM,CAAC,IAAI;QACvB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACtD,eAAe,EAAE,OAAO;QACxB,QAAQ;QACR,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC,CAAC"}
|
package/dist/core/fetch.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { RedirectHop, ResponseInfo } from
|
|
1
|
+
import type { RedirectHop, ResponseInfo } from "./types.js";
|
|
2
2
|
export declare const DEFAULT_USER_AGENT: string;
|
|
3
3
|
export interface FetchOptions {
|
|
4
4
|
timeoutMs?: number;
|
|
@@ -20,8 +20,8 @@ export interface FetchResult {
|
|
|
20
20
|
error?: string;
|
|
21
21
|
}
|
|
22
22
|
/** GET a URL, following redirects manually so the chain is recorded. */
|
|
23
|
-
export declare
|
|
23
|
+
export declare const fetchPage: (url: string, options?: FetchOptions, readBody?: boolean) => Promise<FetchResult>;
|
|
24
24
|
/** Status/redirect check without downloading the body. */
|
|
25
|
-
export declare
|
|
26
|
-
export declare
|
|
27
|
-
export declare
|
|
25
|
+
export declare const probeUrl: (url: string, options?: FetchOptions) => Promise<FetchResult>;
|
|
26
|
+
export declare const toResponseInfo: (r: FetchResult) => ResponseInfo;
|
|
27
|
+
export declare const isHtmlResponse: (r: FetchResult) => boolean;
|