@rivi-software/agentic-seo-check 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +124 -18
  3. package/dist/brand.d.ts +11 -0
  4. package/dist/brand.js +22 -0
  5. package/dist/brand.js.map +1 -0
  6. package/dist/cli.js +174 -86
  7. package/dist/cli.js.map +1 -1
  8. package/dist/core/aeo/audit.d.ts +44 -0
  9. package/dist/core/aeo/audit.js +310 -0
  10. package/dist/core/aeo/audit.js.map +1 -0
  11. package/dist/core/aeo/bots.d.ts +30 -0
  12. package/dist/core/aeo/bots.js +299 -0
  13. package/dist/core/aeo/bots.js.map +1 -0
  14. package/dist/core/aeo/llms-txt.d.ts +8 -0
  15. package/dist/core/aeo/llms-txt.js +88 -0
  16. package/dist/core/aeo/llms-txt.js.map +1 -0
  17. package/dist/core/aeo/page.d.ts +53 -0
  18. package/dist/core/aeo/page.js +312 -0
  19. package/dist/core/aeo/page.js.map +1 -0
  20. package/dist/core/aeo/probe.d.ts +10 -0
  21. package/dist/core/aeo/probe.js +53 -0
  22. package/dist/core/aeo/probe.js.map +1 -0
  23. package/dist/core/aeo/site-rules.d.ts +13 -0
  24. package/dist/core/aeo/site-rules.js +178 -0
  25. package/dist/core/aeo/site-rules.js.map +1 -0
  26. package/dist/core/audit.d.ts +37 -16
  27. package/dist/core/audit.js +389 -249
  28. package/dist/core/audit.js.map +1 -1
  29. package/dist/core/concurrency.d.ts +4 -4
  30. package/dist/core/concurrency.js +33 -23
  31. package/dist/core/concurrency.js.map +1 -1
  32. package/dist/core/crawl.d.ts +9 -5
  33. package/dist/core/crawl.js +271 -123
  34. package/dist/core/crawl.js.map +1 -1
  35. package/dist/core/fetch.d.ts +5 -5
  36. package/dist/core/fetch.js +151 -100
  37. package/dist/core/fetch.js.map +1 -1
  38. package/dist/core/html/answer.d.ts +4 -0
  39. package/dist/core/html/answer.js +198 -0
  40. package/dist/core/html/answer.js.map +1 -0
  41. package/dist/core/html/dom.d.ts +17 -0
  42. package/dist/core/html/dom.js +83 -0
  43. package/dist/core/html/dom.js.map +1 -0
  44. package/dist/core/html/extract.d.ts +2 -2
  45. package/dist/core/html/extract.js +102 -111
  46. package/dist/core/html/extract.js.map +1 -1
  47. package/dist/core/html/structured-data.d.ts +16 -6
  48. package/dist/core/html/structured-data.js +180 -145
  49. package/dist/core/html/structured-data.js.map +1 -1
  50. package/dist/core/html/text.d.ts +14 -14
  51. package/dist/core/html/text.js +110 -107
  52. package/dist/core/html/text.js.map +1 -1
  53. package/dist/core/lighthouse.d.ts +26 -25
  54. package/dist/core/lighthouse.js +181 -105
  55. package/dist/core/lighthouse.js.map +1 -1
  56. package/dist/core/performance-rules.d.ts +2 -2
  57. package/dist/core/performance-rules.js +105 -33
  58. package/dist/core/performance-rules.js.map +1 -1
  59. package/dist/core/render-rules.d.ts +5 -4
  60. package/dist/core/render-rules.js +129 -62
  61. package/dist/core/render-rules.js.map +1 -1
  62. package/dist/core/render.d.ts +2 -2
  63. package/dist/core/render.js +29 -23
  64. package/dist/core/render.js.map +1 -1
  65. package/dist/core/report/markdown.d.ts +4 -2
  66. package/dist/core/report/markdown.js +210 -129
  67. package/dist/core/report/markdown.js.map +1 -1
  68. package/dist/core/robots.d.ts +12 -5
  69. package/dist/core/robots.js +43 -17
  70. package/dist/core/robots.js.map +1 -1
  71. package/dist/core/rules/aeo.d.ts +11 -0
  72. package/dist/core/rules/aeo.js +423 -0
  73. package/dist/core/rules/aeo.js.map +1 -0
  74. package/dist/core/rules/canonical.d.ts +1 -1
  75. package/dist/core/rules/canonical.js +93 -57
  76. package/dist/core/rules/canonical.js.map +1 -1
  77. package/dist/core/rules/content.js +40 -26
  78. package/dist/core/rules/content.js.map +1 -1
  79. package/dist/core/rules/document.js +64 -41
  80. package/dist/core/rules/document.js.map +1 -1
  81. package/dist/core/rules/headings.js +58 -38
  82. package/dist/core/rules/headings.js.map +1 -1
  83. package/dist/core/rules/hreflang.js +70 -38
  84. package/dist/core/rules/hreflang.js.map +1 -1
  85. package/dist/core/rules/images.js +44 -26
  86. package/dist/core/rules/images.js.map +1 -1
  87. package/dist/core/rules/index.d.ts +10 -6
  88. package/dist/core/rules/index.js +43 -36
  89. package/dist/core/rules/index.js.map +1 -1
  90. package/dist/core/rules/indexing.d.ts +2 -2
  91. package/dist/core/rules/indexing.js +48 -26
  92. package/dist/core/rules/indexing.js.map +1 -1
  93. package/dist/core/rules/keyword.js +84 -47
  94. package/dist/core/rules/keyword.js.map +1 -1
  95. package/dist/core/rules/links.js +72 -37
  96. package/dist/core/rules/links.js.map +1 -1
  97. package/dist/core/rules/meta.js +55 -34
  98. package/dist/core/rules/meta.js.map +1 -1
  99. package/dist/core/rules/response.js +75 -41
  100. package/dist/core/rules/response.js.map +1 -1
  101. package/dist/core/rules/rule.d.ts +6 -6
  102. package/dist/core/rules/rule.js +36 -36
  103. package/dist/core/rules/rule.js.map +1 -1
  104. package/dist/core/rules/social.js +86 -45
  105. package/dist/core/rules/social.js.map +1 -1
  106. package/dist/core/rules/structured-data.d.ts +6 -4
  107. package/dist/core/rules/structured-data.js +314 -102
  108. package/dist/core/rules/structured-data.js.map +1 -1
  109. package/dist/core/rules/title.js +48 -33
  110. package/dist/core/rules/title.js.map +1 -1
  111. package/dist/core/score.d.ts +5 -3
  112. package/dist/core/score.js +11 -9
  113. package/dist/core/score.js.map +1 -1
  114. package/dist/core/site-rules.d.ts +3 -3
  115. package/dist/core/site-rules.js +409 -198
  116. package/dist/core/site-rules.js.map +1 -1
  117. package/dist/core/sitemap.d.ts +5 -5
  118. package/dist/core/sitemap.js +109 -64
  119. package/dist/core/sitemap.js.map +1 -1
  120. package/dist/core/static.d.ts +2 -2
  121. package/dist/core/static.js +210 -112
  122. package/dist/core/static.js.map +1 -1
  123. package/dist/core/summary.d.ts +2 -2
  124. package/dist/core/summary.js +11 -6
  125. package/dist/core/summary.js.map +1 -1
  126. package/dist/core/types.d.ts +189 -14
  127. package/dist/core/types.js +15 -8
  128. package/dist/core/types.js.map +1 -1
  129. package/dist/core/url.d.ts +11 -11
  130. package/dist/core/url.js +122 -70
  131. package/dist/core/url.js.map +1 -1
  132. package/dist/index.d.ts +15 -11
  133. package/dist/index.js +11 -9
  134. package/dist/index.js.map +1 -1
  135. package/dist/mcp-bin.js +7 -4
  136. package/dist/mcp-bin.js.map +1 -1
  137. package/dist/mcp.d.ts +3 -3
  138. package/dist/mcp.js +437 -121
  139. package/dist/mcp.js.map +1 -1
  140. package/dist/version.d.ts +0 -1
  141. package/dist/version.js +4 -2
  142. package/dist/version.js.map +1 -1
  143. package/package.json +32 -10
  144. package/skills/seo-audit/SKILL.md +8 -2
@@ -1,23 +1,122 @@
1
- import { extractPage } from './html/extract.js';
2
- import { DEFAULT_USER_AGENT, fetchPage, isHtmlResponse, probeUrl, toResponseInfo } from './fetch.js';
3
- import { loadRobotsTxt } from './robots.js';
4
- import { loadSitemaps } from './sitemap.js';
5
- import { isLocalHost } from './rules/canonical.js';
6
- import { hostOf, isSameSite, looksLikeHtml, normalizeUrl, stripWww } from './url.js';
7
- import { mapConcurrent, sleep } from './concurrency.js';
8
- export async function crawlSite(startInput, options) {
1
+ import { mapConcurrent, sleep } from "./concurrency.js";
2
+ import { DEFAULT_USER_AGENT, fetchPage, isHtmlResponse, probeUrl, toResponseInfo, } from "./fetch.js";
3
+ import { extractPage } from "./html/extract.js";
4
+ import { loadRobotsTxt } from "./robots.js";
5
+ import { isLocalHost } from "./rules/canonical.js";
6
+ import { loadSitemaps } from "./sitemap.js";
7
+ import { hostOf, isSameSite, looksLikeHtml, normalizeUrl, stripWww, } from "./url.js";
8
+ /**
9
+ * Works through a queue that grows while it runs: each crawled page can add the links it found.
10
+ * At most `concurrency` items run at once, and with `delayMs` each slot waits that long after its
11
+ * item before it takes the next one. An item that throws is left out; the rest go on.
12
+ */
13
+ const runQueue = async (queue, concurrency, delayMs, work) => {
14
+ const limit = Math.max(1, concurrency);
15
+ const running = new Set();
16
+ const launch = (item) => {
17
+ const task = (async () => {
18
+ try {
19
+ await work(item);
20
+ }
21
+ catch {
22
+ // The page is left out of the crawl; the rest of the queue goes on.
23
+ }
24
+ if (delayMs > 0) {
25
+ await sleep(delayMs);
26
+ }
27
+ })().finally(() => running.delete(task));
28
+ running.add(task);
29
+ };
30
+ for (;;) {
31
+ while (running.size < limit) {
32
+ const item = queue.shift();
33
+ if (!item) {
34
+ break;
35
+ }
36
+ launch(item);
37
+ }
38
+ if (running.size === 0) {
39
+ return;
40
+ }
41
+ // oxlint-disable-next-line eslint/no-await-in-loop -- the queue grows as pages finish, so what to start next is only known once a slot frees
42
+ await Promise.race(running);
43
+ }
44
+ };
45
+ /** Adds `source` to the set of pages that link to `target`. */
46
+ const addSource = (sources, target, source) => {
47
+ let set = sources.get(target);
48
+ if (!set) {
49
+ set = new Set();
50
+ sources.set(target, set);
51
+ }
52
+ set.add(source);
53
+ };
54
+ const runProbes = async (startUrl, fetchOpts) => {
55
+ const probes = {};
56
+ const url = new URL(startUrl);
57
+ const soft404Url = new URL(`/agentic-seo-probe-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`, url.origin).href;
58
+ const soft = await probeUrl(soft404Url, fetchOpts);
59
+ probes.soft404 = soft.error ? null : { url: soft404Url, status: soft.status };
60
+ if (isLocalHost(startUrl)) {
61
+ return probes;
62
+ }
63
+ if (url.protocol === "https:") {
64
+ const httpUrl = `http://${url.host}/`;
65
+ const res = await probeUrl(httpUrl, { ...fetchOpts, timeoutMs: 10_000 });
66
+ probes.httpRedirect = res.error
67
+ ? null
68
+ : {
69
+ url: httpUrl,
70
+ status: res.status,
71
+ finalUrl: res.finalUrl,
72
+ redirectsToHttps: res.finalUrl.startsWith("https://"),
73
+ };
74
+ }
75
+ const host = url.hostname;
76
+ const variantHost = host.startsWith("www.") ? stripWww(host) : `www.${host}`;
77
+ const variantUrl = `${url.protocol}//${variantHost}/`;
78
+ const res = await probeUrl(variantUrl, { ...fetchOpts, timeoutMs: 10_000 });
79
+ probes.hostVariant = res.error
80
+ ? null
81
+ : {
82
+ url: variantUrl,
83
+ status: res.status,
84
+ finalUrl: res.finalUrl,
85
+ redirectsToMain: (hostOf(res.finalUrl) ?? "") === host,
86
+ };
87
+ return probes;
88
+ };
89
+ /** The progress callback of a caller that does not listen for progress. */
90
+ const ignoreProgress = () => {
91
+ // Nobody to tell.
92
+ };
93
+ export const crawlSite = async (startInput, options) => {
9
94
  const started = performance.now();
10
- const progress = options.onProgress ?? (() => undefined);
95
+ const progress = options.onProgress ?? ignoreProgress;
11
96
  const startUrl = normalizeUrl(startInput) ?? startInput;
12
- const origin = new URL(startUrl).origin;
13
- const startHost = hostOf(startUrl) ?? '';
97
+ const { origin } = new URL(startUrl);
98
+ const startHost = hostOf(startUrl) ?? "";
14
99
  const userAgent = options.fetch.userAgent ?? DEFAULT_USER_AGENT;
15
- progress({ phase: 'robots', message: `Fetching ${origin}/robots.txt` });
100
+ progress({ phase: "robots", message: `Fetching ${origin}/robots.txt` });
16
101
  const robots = await loadRobotsTxt(origin, userAgent, options.fetch);
17
102
  const allows = options.respectRobots ? robots.allows : () => true;
18
- progress({ phase: 'sitemap', message: 'Fetching sitemaps' });
19
- const sitemapCandidates = [...new Set([...robots.info.sitemaps, new URL('/sitemap.xml', origin).href])];
20
- const sitemaps = await loadSitemaps(sitemapCandidates, options.fetch);
103
+ const siteChecks = options.siteChecks ?? true;
104
+ let sitemaps = {
105
+ sources: [],
106
+ urls: [],
107
+ urlCount: 0,
108
+ errors: [],
109
+ };
110
+ if (siteChecks) {
111
+ progress({ phase: "sitemap", message: "Fetching sitemaps" });
112
+ const sitemapCandidates = [
113
+ ...new Set([
114
+ ...robots.info.sitemaps,
115
+ new URL("/sitemap.xml", origin).href,
116
+ ]),
117
+ ];
118
+ sitemaps = await loadSitemaps(sitemapCandidates, options.fetch);
119
+ }
21
120
  const visited = new Set([startUrl]);
22
121
  const discovered = new Set([startUrl]);
23
122
  const linkChecks = new Map();
@@ -27,103 +126,194 @@ export async function crawlSite(startInput, options) {
27
126
  const blocked = [];
28
127
  let limitSkips = 0;
29
128
  const accepts = (url) => {
30
- if (options.includePatterns.length && !options.includePatterns.some((re) => re.test(url)))
129
+ if (options.includePatterns.length &&
130
+ !options.includePatterns.some((re) => re.test(url))) {
31
131
  return false;
32
- if (options.excludePatterns.some((re) => re.test(url)))
132
+ }
133
+ if (options.excludePatterns.some((re) => re.test(url))) {
33
134
  return false;
135
+ }
34
136
  return true;
35
137
  };
36
138
  const queue = [{ url: startUrl, depth: 0 }];
37
- const processItem = async (item) => {
38
- if (!allows(item.url)) {
39
- blocked.push(item.url);
40
- return;
41
- }
42
- const res = await fetchPage(item.url, options.fetch);
43
- const requestedKey = item.url;
44
- const finalKey = normalizeUrl(res.finalUrl) ?? res.finalUrl;
45
- linkChecks.set(requestedKey, { url: item.url, status: res.status, finalUrl: res.finalUrl, redirectChain: res.redirectChain, error: res.error });
46
- if (finalKey !== requestedKey) {
47
- if (!linkChecks.has(finalKey))
48
- linkChecks.set(finalKey, { url: res.finalUrl, status: res.status, finalUrl: res.finalUrl, redirectChain: [], error: res.error });
49
- if (visited.has(finalKey) && records.some((r) => r.url === finalKey))
50
- return;
51
- visited.add(finalKey);
52
- if (!isSameSite(finalKey, startHost, options.includeSubdomains))
53
- return; // redirected off-site
54
- }
55
- progress({ phase: 'crawl', message: `${res.status ?? 'ERR'} ${res.finalUrl}`, crawled: records.length + 1, discovered: discovered.size, url: res.finalUrl });
56
- if (res.error || typeof res.body !== 'string' || !isHtmlResponse(res)) {
57
- if (res.status !== null && res.status < 400 && !res.error && typeof res.body === 'string' && !isHtmlResponse(res))
58
- return; // non-HTML asset
59
- // Record error pages so page-level findings (response.error-status) appear for the start URL.
60
- if (item.depth === 0 || res.status === null) {
61
- const page = extractPage(res.body ?? '', { url: res.finalUrl, siteHost: startHost, includeSubdomains: options.includeSubdomains });
62
- records.push({ url: finalKey, requestedUrl: item.url, page, response: toResponseInfo(res), depth: item.depth });
63
- }
64
- return;
65
- }
66
- const page = extractPage(res.body, { url: res.finalUrl, siteHost: startHost, includeSubdomains: options.includeSubdomains });
67
- records.push({ url: finalKey, requestedUrl: item.url, page, response: toResponseInfo(res), depth: item.depth });
68
- for (const link of page.links) {
69
- if (!link.url)
70
- continue;
71
- const target = normalizeUrl(link.url);
72
- if (!target)
139
+ /** Queues the internal links a crawled page has, and remembers every link's sources. */
140
+ const followLinks = (links, from, depth) => {
141
+ for (const link of links) {
142
+ const target = link.url ? normalizeUrl(link.url) : null;
143
+ if (!target) {
73
144
  continue;
145
+ }
74
146
  if (!link.internal) {
75
- if (!externalSources.has(target))
76
- externalSources.set(target, new Set());
77
- externalSources.get(target).add(finalKey);
147
+ addSource(externalSources, target, from);
78
148
  continue;
79
149
  }
80
- if (!linkSources.has(target))
81
- linkSources.set(target, new Set());
82
- linkSources.get(target).add(finalKey);
83
- if (!looksLikeHtml(target) || !accepts(target))
84
- continue;
85
- if (discovered.has(target))
150
+ addSource(linkSources, target, from);
151
+ if (!looksLikeHtml(target) ||
152
+ !accepts(target) ||
153
+ discovered.has(target)) {
86
154
  continue;
155
+ }
87
156
  discovered.add(target);
88
- if (item.depth + 1 > options.maxDepth)
157
+ if (depth + 1 > options.maxDepth) {
89
158
  continue;
159
+ }
90
160
  if (visited.size >= options.maxPages) {
91
- limitSkips++;
161
+ limitSkips += 1;
92
162
  continue;
93
163
  }
94
164
  visited.add(target);
95
- queue.push({ url: target, depth: item.depth + 1 });
165
+ queue.push({ url: target, depth: depth + 1 });
96
166
  }
97
167
  };
168
+ /**
169
+ * Records where a redirected request ended. False when there is nothing more to crawl there:
170
+ * the redirect led to a page crawled already, or off the site.
171
+ */
172
+ const keepsRedirectTarget = (res, finalKey) => {
173
+ if (!linkChecks.has(finalKey)) {
174
+ linkChecks.set(finalKey, {
175
+ url: res.finalUrl,
176
+ status: res.status,
177
+ finalUrl: res.finalUrl,
178
+ redirectChain: [],
179
+ error: res.error,
180
+ });
181
+ }
182
+ if (visited.has(finalKey) && records.some((r) => r.url === finalKey)) {
183
+ return false;
184
+ }
185
+ visited.add(finalKey);
186
+ return isSameSite(finalKey, startHost, options.includeSubdomains);
187
+ };
188
+ /** A response that is no HTML page: an asset is skipped, an error page may still be recorded. */
189
+ const recordUnreadable = (item, res, finalKey) => {
190
+ const isAsset = res.status !== null &&
191
+ res.status < 400 &&
192
+ !res.error &&
193
+ res.body !== null &&
194
+ !isHtmlResponse(res);
195
+ // Record error pages so page-level findings (response.error-status) appear for the start URL.
196
+ if (isAsset || (item.depth !== 0 && res.status !== null)) {
197
+ return;
198
+ }
199
+ const page = extractPage(res.body ?? "", {
200
+ url: res.finalUrl,
201
+ siteHost: startHost,
202
+ includeSubdomains: options.includeSubdomains,
203
+ });
204
+ records.push({
205
+ url: finalKey,
206
+ requestedUrl: item.url,
207
+ page,
208
+ response: toResponseInfo(res),
209
+ depth: item.depth,
210
+ });
211
+ };
212
+ const processItem = async (item) => {
213
+ if (!allows(item.url)) {
214
+ blocked.push(item.url);
215
+ return;
216
+ }
217
+ const res = await fetchPage(item.url, options.fetch);
218
+ const requestedKey = item.url;
219
+ const finalKey = normalizeUrl(res.finalUrl) ?? res.finalUrl;
220
+ linkChecks.set(requestedKey, {
221
+ url: item.url,
222
+ status: res.status,
223
+ finalUrl: res.finalUrl,
224
+ redirectChain: res.redirectChain,
225
+ error: res.error,
226
+ });
227
+ if (finalKey !== requestedKey && !keepsRedirectTarget(res, finalKey)) {
228
+ return;
229
+ }
230
+ progress({
231
+ phase: "crawl",
232
+ message: `${res.status ?? "ERR"} ${res.finalUrl}`,
233
+ crawled: records.length + 1,
234
+ discovered: discovered.size,
235
+ url: res.finalUrl,
236
+ });
237
+ if (res.error || res.body === null || !isHtmlResponse(res)) {
238
+ recordUnreadable(item, res, finalKey);
239
+ return;
240
+ }
241
+ const page = extractPage(res.body, {
242
+ url: res.finalUrl,
243
+ siteHost: startHost,
244
+ includeSubdomains: options.includeSubdomains,
245
+ });
246
+ records.push({
247
+ url: finalKey,
248
+ requestedUrl: item.url,
249
+ page,
250
+ response: toResponseInfo(res),
251
+ depth: item.depth,
252
+ });
253
+ followLinks(page.links, finalKey, item.depth);
254
+ };
98
255
  await runQueue(queue, options.concurrency, options.delayMs, processItem);
99
256
  /* Check internal link targets that were discovered but not crawled (limits, depth, non-HTML). */
100
- progress({ phase: 'links', message: 'Checking internal link targets' });
101
- const toCheck = [...linkSources.keys()].filter((u) => !linkChecks.has(u) && allows(u)).slice(0, options.maxLinkChecks);
257
+ progress({ phase: "links", message: "Checking internal link targets" });
258
+ const toCheck = [...linkSources.keys()]
259
+ .filter((u) => !linkChecks.has(u) && allows(u))
260
+ .slice(0, options.maxLinkChecks);
102
261
  const sitemapExtra = sitemaps.urls
103
262
  .map((u) => normalizeUrl(u) ?? u)
104
- .filter((u) => isSameSite(u, startHost, options.includeSubdomains) && !linkChecks.has(u) && !toCheck.includes(u) && allows(u))
263
+ .filter((u) => isSameSite(u, startHost, options.includeSubdomains) &&
264
+ !linkChecks.has(u) &&
265
+ !toCheck.includes(u) &&
266
+ allows(u))
105
267
  .slice(0, Math.max(0, options.maxLinkChecks - toCheck.length));
106
- for (const u of [...linkSources.keys()].filter((u) => !allows(u)))
107
- if (!blocked.includes(u))
108
- blocked.push(u);
268
+ for (const target of [...linkSources.keys()].filter((u) => !allows(u))) {
269
+ if (!blocked.includes(target)) {
270
+ blocked.push(target);
271
+ }
272
+ }
109
273
  await mapConcurrent([...toCheck, ...sitemapExtra], options.concurrency, async (u) => {
110
274
  const res = await probeUrl(u, options.fetch);
111
- linkChecks.set(u, { url: u, status: res.status, finalUrl: res.finalUrl, redirectChain: res.redirectChain, error: res.error });
112
- if (options.delayMs)
275
+ linkChecks.set(u, {
276
+ url: u,
277
+ status: res.status,
278
+ finalUrl: res.finalUrl,
279
+ redirectChain: res.redirectChain,
280
+ error: res.error,
281
+ });
282
+ if (options.delayMs) {
113
283
  await sleep(options.delayMs);
284
+ }
114
285
  });
115
286
  let externalChecks;
116
287
  if (options.checkExternalLinks) {
117
- progress({ phase: 'links', message: `Checking ${Math.min(externalSources.size, options.maxLinkChecks)} external links` });
118
- externalChecks = new Map();
288
+ progress({
289
+ phase: "links",
290
+ message: `Checking ${Math.min(externalSources.size, options.maxLinkChecks)} external links`,
291
+ });
292
+ const checks = new Map();
293
+ externalChecks = checks;
119
294
  const targets = [...externalSources.keys()].slice(0, options.maxLinkChecks);
120
295
  await mapConcurrent(targets, Math.min(options.concurrency, 4), async (u) => {
121
- const res = await probeUrl(u, { ...options.fetch, timeoutMs: Math.min(options.fetch.timeoutMs ?? 15_000, 10_000) });
122
- externalChecks.set(u, { url: u, status: res.status, finalUrl: res.finalUrl, redirectChain: res.redirectChain, error: res.error });
296
+ const res = await probeUrl(u, {
297
+ ...options.fetch,
298
+ timeoutMs: Math.min(options.fetch.timeoutMs ?? 15_000, 10_000),
299
+ });
300
+ checks.set(u, {
301
+ url: u,
302
+ status: res.status,
303
+ finalUrl: res.finalUrl,
304
+ redirectChain: res.redirectChain,
305
+ error: res.error,
306
+ });
307
+ });
308
+ }
309
+ let probes = {};
310
+ if (siteChecks) {
311
+ progress({
312
+ phase: "probes",
313
+ message: "Probing soft-404 and host redirects",
123
314
  });
315
+ probes = await runProbes(startUrl, options.fetch);
124
316
  }
125
- progress({ phase: 'probes', message: 'Probing soft-404 and host redirects' });
126
- const probes = await runProbes(startUrl, options.fetch);
127
317
  const stats = {
128
318
  pagesCrawled: records.length,
129
319
  pagesDiscovered: discovered.size,
@@ -142,54 +332,12 @@ export async function crawlSite(startInput, options) {
142
332
  externalChecks,
143
333
  externalSources: options.checkExternalLinks ? externalSources : undefined,
144
334
  robots: robots.info,
335
+ robotsText: robots.text,
145
336
  robotsAllows: robots.info.found ? robots.allows : null,
146
337
  blockedByRobots: blocked,
147
338
  sitemaps,
148
339
  probes,
149
340
  stats,
150
341
  };
151
- }
152
- async function runQueue(queue, concurrency, delayMs, work) {
153
- let active = 0;
154
- await new Promise((resolve) => {
155
- const pump = () => {
156
- while (active < Math.max(1, concurrency) && queue.length) {
157
- const item = queue.shift();
158
- active++;
159
- work(item)
160
- .catch(() => undefined)
161
- .finally(() => {
162
- active--;
163
- if (delayMs > 0)
164
- setTimeout(pump, delayMs);
165
- else
166
- pump();
167
- });
168
- }
169
- if (active === 0 && queue.length === 0)
170
- resolve();
171
- };
172
- pump();
173
- });
174
- }
175
- async function runProbes(startUrl, fetchOpts) {
176
- const probes = {};
177
- const url = new URL(startUrl);
178
- const soft404Url = new URL(`/agentic-seo-probe-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`, url.origin).href;
179
- const soft = await probeUrl(soft404Url, fetchOpts);
180
- probes.soft404 = soft.error ? null : { url: soft404Url, status: soft.status };
181
- if (isLocalHost(startUrl))
182
- return probes;
183
- if (url.protocol === 'https:') {
184
- const httpUrl = `http://${url.host}/`;
185
- const res = await probeUrl(httpUrl, { ...fetchOpts, timeoutMs: 10_000 });
186
- probes.httpRedirect = res.error ? null : { url: httpUrl, status: res.status, finalUrl: res.finalUrl, redirectsToHttps: res.finalUrl.startsWith('https://') };
187
- }
188
- const host = url.hostname;
189
- const variantHost = host.startsWith('www.') ? stripWww(host) : `www.${host}`;
190
- const variantUrl = `${url.protocol}//${variantHost}/`;
191
- const res = await probeUrl(variantUrl, { ...fetchOpts, timeoutMs: 10_000 });
192
- probes.hostVariant = res.error ? null : { url: variantUrl, status: res.status, finalUrl: res.finalUrl, redirectsToMain: (hostOf(res.finalUrl) ?? '') === host };
193
- return probes;
194
- }
342
+ };
195
343
  //# sourceMappingURL=crawl.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,cAAc,EAAqB,MAAM,YAAY,CAAC;AACxH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAGnD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AA8CxD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,UAAkB,EAAE,OAAqB;IACvE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;IACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEhE,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,MAAM,aAAa,EAAE,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;IAElE,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC7D,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxG,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACvD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE;QACvC,IAAI,OAAO,CAAC,eAAe,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACxG,IAAI,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACrE,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEzD,MAAM,WAAW,GAAG,KAAK,EAAE,IAAe,EAAiB,EAAE;QAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO;QACT,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC;QAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;QAC5D,UAAU,CAAC,GAAG,CAAC,YAAY,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAChJ,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAChK,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC;gBAAE,OAAO;YAC7E,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC;gBAAE,OAAO,CAAC,sBAAsB;QACjG,CAAC;QACD,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC7J,IAAI,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YACtE,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;gBAAE,OAAO,CAAC,iBAAiB;YAC5I,8FAA8F;YAC9F,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;gBACnI,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAClH,CAAC;YACD,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC7H,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAEhH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,SAAS;YACxB,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,CAAC,MAAM;gBAAE,SAAS;YACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC;oBAAE,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;gBACzE,eAAe,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YACjE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,SAAS;YACzD,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAS;YACrC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvB,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ;gBAAE,SAAS;YAChD,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrC,UAAU,EAAE,CAAC;gBACb,SAAS;YACX,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEzE,iGAAiG;IACjG,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACvH,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;SAC7H,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACjE,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,aAAa,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,YAAY,CAAC,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAClF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAC9H,IAAI,OAAO,CAAC,OAAO;YAAE,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,IAAI,cAAkD,CAAC;IACvD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC1H,cAAc,GAAG,IAAI,GAAG,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC5E,MAAM,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;YACzE,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;YACpH,cAAe,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QACrI,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAExD,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,eAAe,EAAE,UAAU,CAAC,IAAI;QAChC,YAAY,EAAE,UAAU,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,CAAC;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,YAAY,EAAE,UAAU,GAAG,CAAC;QAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACnD,eAAe,EAAE,OAAO,CAAC,aAAa;KACvC,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,OAAO;QACP,UAAU;QACV,WAAW;QACX,cAAc;QACd,eAAe,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;QACzE,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACtD,eAAe,EAAE,OAAO;QACxB,QAAQ;QACR,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,KAAkB,EAAE,WAAmB,EAAE,OAAe,EAAE,IAAwC;IACxH,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,MAAM,IAAI,GAAG,GAAS,EAAE;YACtB,OAAO,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACzD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;gBAC5B,MAAM,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC;qBACP,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;qBACtB,OAAO,CAAC,GAAG,EAAE;oBACZ,MAAM,EAAE,CAAC;oBACT,IAAI,OAAO,GAAG,CAAC;wBAAE,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;;wBACtC,IAAI,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;YACD,IAAI,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;QACpD,CAAC,CAAC;QACF,IAAI,EAAE,CAAC;IACT,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,SAAuB;IAChE,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;IACvI,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAE9E,IAAI,WAAW,CAAC,QAAQ,CAAC;QAAE,OAAO,MAAM,CAAC;IAEzC,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/J,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,WAAW,GAAG,CAAC;IACtD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,eAAe,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;IAChK,OAAO,MAAM,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,kBAAkB,EAClB,SAAS,EACT,cAAc,EACd,QAAQ,EACR,cAAc,GACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAO5C,OAAO,EACL,MAAM,EACN,UAAU,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,GACT,MAAM,UAAU,CAAC;AA4DlB;;;;GAIG;AACH,MAAM,QAAQ,GAAG,KAAK,EACpB,KAAkB,EAClB,WAAmB,EACnB,OAAe,EACf,IAAwC,EACzB,EAAE;IACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IAEzC,MAAM,MAAM,GAAG,CAAC,IAAe,EAAQ,EAAE;QACvC,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE;YACvB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;YAAC,MAAM,CAAC;gBACP,oEAAoE;YACtE,CAAC;YAED,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEzC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC;IAEF,SAAS,CAAC;QACR,OAAO,OAAO,CAAC,IAAI,GAAG,KAAK,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAE3B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM;YACR,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,6IAA6I;QAC7I,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC,CAAC;AAEF,+DAA+D;AAC/D,MAAM,SAAS,GAAG,CAChB,OAAiC,EACjC,MAAc,EACd,MAAc,EACR,EAAE;IACR,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EACrB,QAAgB,EAChB,SAAuB,EACA,EAAE;IACzB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAE9B,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EACzF,GAAG,CAAC,MAAM,CACX,CAAC,IAAI,CAAC;IAEP,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAE9E,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK;YAC7B,CAAC,CAAC,IAAI;YACN,CAAC,CAAC;gBACE,GAAG,EAAE,OAAO;gBACZ,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;aACtD,CAAC;IACR,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,WAAW,GAAG,CAAC;IACtD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,KAAK;QAC5B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC;YACE,GAAG,EAAE,UAAU;YACf,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,eAAe,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI;SACvD,CAAC;IAEN,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,cAAc,GAAG,GAAS,EAAE;IAChC,kBAAkB;AACpB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,UAAkB,EAClB,OAAqB,EACC,EAAE;IACxB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,cAAc,CAAC;IACtD,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;IACxD,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEhE,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,MAAM,aAAa,EAAE,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;IAElE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;IAE9C,IAAI,QAAQ,GAAgB;QAC1B,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,IAAI,UAAU,EAAE,CAAC;QACf,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;QAE7D,MAAM,iBAAiB,GAAG;YACxB,GAAG,IAAI,GAAG,CAAC;gBACT,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ;gBACvB,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI;aACrC,CAAC;SACH,CAAC;QAEF,QAAQ,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAClE,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACvD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE;QACvC,IACE,OAAO,CAAC,eAAe,CAAC,MAAM;YAC9B,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACnD,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEzD,wFAAwF;IACxF,MAAM,WAAW,GAAG,CAClB,KAAiB,EACjB,IAAY,EACZ,KAAa,EACP,EAAE;QACR,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAExD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,SAAS,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;YAErC,IACE,CAAC,aAAa,CAAC,MAAM,CAAC;gBACtB,CAAC,OAAO,CAAC,MAAM,CAAC;gBAChB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EACtB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAEvB,IAAI,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACjC,SAAS;YACX,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrC,UAAU,IAAI,CAAC,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC,CAAC;IAEF;;;OAGG;IACH,MAAM,mBAAmB,GAAG,CAAC,GAAgB,EAAE,QAAgB,EAAW,EAAE;QAC1E,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACvB,GAAG,EAAE,GAAG,CAAC,QAAQ;gBACjB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,EAAE;gBACjB,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtB,OAAO,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpE,CAAC,CAAC;IAEF,iGAAiG;IACjG,MAAM,gBAAgB,GAAG,CACvB,IAAe,EACf,GAAgB,EAChB,QAAgB,EACV,EAAE;QACR,MAAM,OAAO,GACX,GAAG,CAAC,MAAM,KAAK,IAAI;YACnB,GAAG,CAAC,MAAM,GAAG,GAAG;YAChB,CAAC,GAAG,CAAC,KAAK;YACV,GAAG,CAAC,IAAI,KAAK,IAAI;YACjB,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAEvB,8FAA8F;QAC9F,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC;YACzD,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE;YACvC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,KAAK,EAAE,IAAe,EAAiB,EAAE;QAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEvB,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC;QAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;QAE5D,UAAU,CAAC,GAAG,CAAC,YAAY,EAAE;YAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,QAAQ,KAAK,YAAY,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO;QACT,CAAC;QAED,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE;YACjD,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;YAC3B,UAAU,EAAE,UAAU,CAAC,IAAI;YAC3B,GAAG,EAAE,GAAG,CAAC,QAAQ;SAClB,CAAC,CAAC;QAEH,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3D,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;YAEtC,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE;YACjC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;QAEH,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEzE,iGAAiG;IACjG,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IAExE,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;SACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;SAC9C,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAEnC,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAChC,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,UAAU,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC;QACnD,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAClB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpB,MAAM,CAAC,CAAC,CAAC,CACZ;SACA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAEjE,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,aAAa,CACjB,CAAC,GAAG,OAAO,EAAE,GAAG,YAAY,CAAC,EAC7B,OAAO,CAAC,WAAW,EACnB,KAAK,EAAE,CAAC,EAAE,EAAE;QACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE;YAChB,GAAG,EAAE,CAAC;YACN,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CACF,CAAC;IAEF,IAAI,cAAkD,CAAC;IAEvD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,YAAY,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,iBAAiB;SAC5F,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC5C,cAAc,GAAG,MAAM,CAAC;QACxB,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC5E,MAAM,aAAa,CACjB,OAAO,EACP,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAChC,KAAK,EAAE,CAAC,EAAE,EAAE;YACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE;gBAC5B,GAAG,OAAO,CAAC,KAAK;gBAChB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;aAC/D,CAAC,CAAC;YAEH,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE;gBACZ,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC,CACF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,GAAiB,EAAE,CAAC;IAE9B,IAAI,UAAU,EAAE,CAAC;QACf,QAAQ,CAAC;YACP,KAAK,EAAE,QAAQ;YACf,OAAO,EAAE,qCAAqC;SAC/C,CAAC,CAAC;QACH,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,eAAe,EAAE,UAAU,CAAC,IAAI;QAChC,YAAY,EAAE,UAAU,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,CAAC;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,YAAY,EAAE,UAAU,GAAG,CAAC;QAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACnD,eAAe,EAAE,OAAO,CAAC,aAAa;KACvC,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,OAAO;QACP,UAAU;QACV,WAAW;QACX,cAAc;QACd,eAAe,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;QACzE,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,UAAU,EAAE,MAAM,CAAC,IAAI;QACvB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACtD,eAAe,EAAE,OAAO;QACxB,QAAQ;QACR,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC,CAAC"}
@@ -1,4 +1,4 @@
1
- import type { RedirectHop, ResponseInfo } from './types.js';
1
+ import type { RedirectHop, ResponseInfo } from "./types.js";
2
2
  export declare const DEFAULT_USER_AGENT: string;
3
3
  export interface FetchOptions {
4
4
  timeoutMs?: number;
@@ -20,8 +20,8 @@ export interface FetchResult {
20
20
  error?: string;
21
21
  }
22
22
  /** GET a URL, following redirects manually so the chain is recorded. */
23
- export declare function fetchPage(url: string, options?: FetchOptions, readBody?: boolean): Promise<FetchResult>;
23
+ export declare const fetchPage: (url: string, options?: FetchOptions, readBody?: boolean) => Promise<FetchResult>;
24
24
  /** Status/redirect check without downloading the body. */
25
- export declare function probeUrl(url: string, options?: FetchOptions): Promise<FetchResult>;
26
- export declare function toResponseInfo(r: FetchResult): ResponseInfo;
27
- export declare function isHtmlResponse(r: FetchResult): boolean;
25
+ export declare const probeUrl: (url: string, options?: FetchOptions) => Promise<FetchResult>;
26
+ export declare const toResponseInfo: (r: FetchResult) => ResponseInfo;
27
+ export declare const isHtmlResponse: (r: FetchResult) => boolean;