@mdream/crawl 2.0.0-beta.1 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +194 -88
- package/dist/_chunks/crawl.mjs +272 -36
- package/dist/cli.mjs +328 -358
- package/dist/index.d.mts +64 -59
- package/package.json +15 -5
package/dist/_chunks/crawl.mjs
CHANGED
|
@@ -5,7 +5,7 @@ import { createHooks } from "hookable";
|
|
|
5
5
|
import { htmlToMarkdown } from "mdream";
|
|
6
6
|
import { ofetch } from "ofetch";
|
|
7
7
|
import { dirname, join, normalize, resolve } from "pathe";
|
|
8
|
-
import { withHttps } from "ufo";
|
|
8
|
+
import { hasProtocol, withHttps } from "ufo";
|
|
9
9
|
import picomatch from "picomatch";
|
|
10
10
|
import { getDomain } from "tldts";
|
|
11
11
|
import * as p from "@clack/prompts";
|
|
@@ -164,14 +164,14 @@ function matchesGlobPattern(url, parsedPattern, allowSubdomains = false) {
|
|
|
164
164
|
}
|
|
165
165
|
}
|
|
166
166
|
function getStartingUrl(parsedPattern) {
|
|
167
|
-
|
|
167
|
+
const baseUrl = hasProtocol(parsedPattern.baseUrl) ? parsedPattern.baseUrl : withHttps(parsedPattern.baseUrl);
|
|
168
|
+
if (!parsedPattern.isGlob) return baseUrl;
|
|
168
169
|
const pattern = parsedPattern.pattern;
|
|
169
170
|
const firstGlobIndex = pattern.search(GLOB_CHAR_RE);
|
|
170
|
-
if (firstGlobIndex === -1) return
|
|
171
|
+
if (firstGlobIndex === -1) return baseUrl + pattern;
|
|
171
172
|
const beforeGlob = pattern.substring(0, firstGlobIndex);
|
|
172
173
|
const lastSlash = beforeGlob.lastIndexOf("/");
|
|
173
|
-
|
|
174
|
-
return withHttps(parsedPattern.baseUrl + pathBeforeGlob);
|
|
174
|
+
return baseUrl + (lastSlash >= 0 ? beforeGlob.substring(0, lastSlash + 1) : "/");
|
|
175
175
|
}
|
|
176
176
|
function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
|
|
177
177
|
if (!excludePatterns || excludePatterns.length === 0) return false;
|
|
@@ -236,6 +236,217 @@ function resolveLogger(options) {
|
|
|
236
236
|
if (options.logger) return options.logger;
|
|
237
237
|
return options.silent ? silentLogger : createClackLogger();
|
|
238
238
|
}
|
|
239
|
+
const CRAWL_ARTIFACTS = [
|
|
240
|
+
"llms.txt",
|
|
241
|
+
"llms-full.txt",
|
|
242
|
+
"markdown"
|
|
243
|
+
];
|
|
244
|
+
const CRAWL_DRIVERS = ["http", "playwright"];
|
|
245
|
+
const CRAWL_DEFAULTS = {
|
|
246
|
+
output: "output",
|
|
247
|
+
depth: 3,
|
|
248
|
+
driver: "http",
|
|
249
|
+
artifacts: CRAWL_ARTIFACTS,
|
|
250
|
+
boilerplateThreshold: DEFAULT_BOILERPLATE_THRESHOLD
|
|
251
|
+
};
|
|
252
|
+
const OPTION_KEYS = {
|
|
253
|
+
urls: true,
|
|
254
|
+
output: true,
|
|
255
|
+
depth: true,
|
|
256
|
+
maxPages: true,
|
|
257
|
+
sitemap: true,
|
|
258
|
+
artifacts: true,
|
|
259
|
+
siteName: true,
|
|
260
|
+
description: true,
|
|
261
|
+
origin: true,
|
|
262
|
+
driver: true,
|
|
263
|
+
useChrome: true,
|
|
264
|
+
exclude: true,
|
|
265
|
+
crawlDelay: true,
|
|
266
|
+
skipSitemap: true,
|
|
267
|
+
allowSubdomains: true,
|
|
268
|
+
stripBoilerplate: true,
|
|
269
|
+
boilerplateThreshold: true,
|
|
270
|
+
verbose: true,
|
|
271
|
+
silent: true,
|
|
272
|
+
logger: true,
|
|
273
|
+
hooks: true,
|
|
274
|
+
globPatterns: true
|
|
275
|
+
};
|
|
276
|
+
const ARTIFACTS_HINT = "Use \"artifacts\" instead, for example artifacts: ['llms.txt', 'markdown'].";
|
|
277
|
+
const REMOVED_OPTIONS = {
|
|
278
|
+
maxRequestsPerCrawl: "Use \"maxPages\" instead.",
|
|
279
|
+
maxDepth: "Use \"depth\" instead.",
|
|
280
|
+
followLinks: "Use \"depth\" instead. Set depth to 0 to process only the given URLs.",
|
|
281
|
+
sitemapUrls: "Use \"sitemap\" instead.",
|
|
282
|
+
generateLlmsTxt: ARTIFACTS_HINT,
|
|
283
|
+
generateLlmsFullTxt: ARTIFACTS_HINT,
|
|
284
|
+
generateIndividualMd: ARTIFACTS_HINT,
|
|
285
|
+
siteNameOverride: "Use \"siteName\" instead.",
|
|
286
|
+
descriptionOverride: "Use \"description\" instead.",
|
|
287
|
+
outputDir: "Use \"output\" instead.",
|
|
288
|
+
chunkSize: "Remove it. It had no effect.",
|
|
289
|
+
onPage: "Use hooks: { 'crawl:page': fn } instead."
|
|
290
|
+
};
|
|
291
|
+
function fail(message) {
|
|
292
|
+
throw new TypeError(message);
|
|
293
|
+
}
|
|
294
|
+
function show(value) {
|
|
295
|
+
return typeof value === "string" ? JSON.stringify(value) : String(value);
|
|
296
|
+
}
|
|
297
|
+
function assertCrawlOptionKeys(options) {
|
|
298
|
+
const lines = [];
|
|
299
|
+
let hasTypo = false;
|
|
300
|
+
for (const key in options) {
|
|
301
|
+
if (options[key] === void 0 || key in OPTION_KEYS) continue;
|
|
302
|
+
const hint = REMOVED_OPTIONS[key];
|
|
303
|
+
if (hint === void 0) hasTypo = true;
|
|
304
|
+
lines.push(hint === void 0 ? `Unknown crawl option ${show(key)}.` : `Unknown crawl option ${show(key)}. ${hint}`);
|
|
305
|
+
}
|
|
306
|
+
if (lines.length === 0) return;
|
|
307
|
+
if (hasTypo) lines.push(`Valid options: ${Object.keys(OPTION_KEYS).join(", ")}.`);
|
|
308
|
+
fail(lines.join("\n"));
|
|
309
|
+
}
|
|
310
|
+
function validateCrawlUrl(url) {
|
|
311
|
+
const globError = validateGlobPattern(url);
|
|
312
|
+
if (globError) return globError;
|
|
313
|
+
if (!parseUrlPattern(url).isGlob && !URL.canParse(withHttps(url))) return `Invalid URL: ${show(url)}.`;
|
|
314
|
+
}
|
|
315
|
+
function readBoolean(raw, key, fallback) {
|
|
316
|
+
const value = raw[key];
|
|
317
|
+
if (value === void 0) return fallback;
|
|
318
|
+
if (typeof value !== "boolean") fail(`"${key}" must be true or false. You gave ${show(value)}.`);
|
|
319
|
+
return value;
|
|
320
|
+
}
|
|
321
|
+
function readString(raw, key) {
|
|
322
|
+
const value = raw[key];
|
|
323
|
+
if (value !== void 0 && typeof value !== "string") fail(`"${key}" must be a string. You gave ${show(value)}.`);
|
|
324
|
+
return value;
|
|
325
|
+
}
|
|
326
|
+
function readStrings(raw, key) {
|
|
327
|
+
const value = raw[key];
|
|
328
|
+
if (value === void 0) return void 0;
|
|
329
|
+
if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) fail(`"${key}" must be an array of strings.`);
|
|
330
|
+
return value;
|
|
331
|
+
}
|
|
332
|
+
function readObject(raw, key) {
|
|
333
|
+
const value = raw[key];
|
|
334
|
+
if (value !== void 0 && (typeof value !== "object" || value === null)) fail(`"${key}" must be an object.`);
|
|
335
|
+
return value;
|
|
336
|
+
}
|
|
337
|
+
function readUrls(raw) {
|
|
338
|
+
const urls = readStrings(raw, "urls");
|
|
339
|
+
if (!urls || urls.length === 0) fail("\"urls\" must be a non-empty array of URLs.");
|
|
340
|
+
for (const url of urls) {
|
|
341
|
+
const error = validateCrawlUrl(url);
|
|
342
|
+
if (error) fail(error);
|
|
343
|
+
}
|
|
344
|
+
return urls;
|
|
345
|
+
}
|
|
346
|
+
function readSitemap(raw) {
|
|
347
|
+
const value = raw.sitemap;
|
|
348
|
+
if (value === void 0) return [];
|
|
349
|
+
const list = typeof value === "string" ? [value] : value;
|
|
350
|
+
if (!Array.isArray(list) || !list.every((item) => typeof item === "string")) fail("\"sitemap\" must be a URL or an array of URLs.");
|
|
351
|
+
return list.map((item) => {
|
|
352
|
+
const url = withHttps(item);
|
|
353
|
+
if (!URL.canParse(url)) fail(`"sitemap" has an invalid URL: ${show(item)}.`);
|
|
354
|
+
return url;
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
function readArtifacts(raw) {
|
|
358
|
+
const value = raw.artifacts;
|
|
359
|
+
if (value === void 0) return [...CRAWL_DEFAULTS.artifacts];
|
|
360
|
+
const valid = `Valid artifacts: ${CRAWL_ARTIFACTS.join(", ")}.`;
|
|
361
|
+
if (!Array.isArray(value)) fail(`"artifacts" must be an array. ${valid}`);
|
|
362
|
+
for (const artifact of value) if (!CRAWL_ARTIFACTS.includes(artifact)) fail(`Invalid artifact ${show(artifact)}. ${valid}`);
|
|
363
|
+
return CRAWL_ARTIFACTS.filter((artifact) => value.includes(artifact));
|
|
364
|
+
}
|
|
365
|
+
function readDriver(raw) {
|
|
366
|
+
const value = raw.driver;
|
|
367
|
+
if (value === void 0) return CRAWL_DEFAULTS.driver;
|
|
368
|
+
if (!CRAWL_DRIVERS.includes(value)) fail(`"driver" must be "http" or "playwright". You gave ${show(value)}.`);
|
|
369
|
+
return value;
|
|
370
|
+
}
|
|
371
|
+
function readDepth(raw) {
|
|
372
|
+
const value = raw.depth;
|
|
373
|
+
if (value === void 0) return CRAWL_DEFAULTS.depth;
|
|
374
|
+
if (!Number.isInteger(value) || value < 0 || value > 10) fail(`"depth" must be an integer from 0 to 10. You gave ${show(value)}.`);
|
|
375
|
+
return value;
|
|
376
|
+
}
|
|
377
|
+
function readMaxPages(raw) {
|
|
378
|
+
const value = raw.maxPages;
|
|
379
|
+
if (value === void 0) return Number.POSITIVE_INFINITY;
|
|
380
|
+
if (!Number.isInteger(value) || value < 1) fail(`"maxPages" must be an integer of 1 or more. You gave ${show(value)}.`);
|
|
381
|
+
return value;
|
|
382
|
+
}
|
|
383
|
+
function readCrawlDelay(raw) {
|
|
384
|
+
const value = raw.crawlDelay;
|
|
385
|
+
if (value === void 0) return void 0;
|
|
386
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) fail(`"crawlDelay" must be a number of seconds, 0 or more. You gave ${show(value)}.`);
|
|
387
|
+
return value;
|
|
388
|
+
}
|
|
389
|
+
function readBoilerplateThreshold(raw) {
|
|
390
|
+
const value = raw.boilerplateThreshold;
|
|
391
|
+
if (value === void 0) return CRAWL_DEFAULTS.boilerplateThreshold;
|
|
392
|
+
if (typeof value !== "number" || !(value > 0 && value <= 1)) fail(`"boilerplateThreshold" must be a number greater than 0 and at most 1. You gave ${show(value)}.`);
|
|
393
|
+
return value;
|
|
394
|
+
}
|
|
395
|
+
function readExclude(raw) {
|
|
396
|
+
const exclude = readStrings(raw, "exclude") ?? [];
|
|
397
|
+
for (const pattern of exclude) {
|
|
398
|
+
const error = validateGlobPattern(pattern);
|
|
399
|
+
if (error) fail(`"exclude" has an invalid pattern. ${error}`);
|
|
400
|
+
}
|
|
401
|
+
return exclude;
|
|
402
|
+
}
|
|
403
|
+
function resolveCrawlOptions(options) {
|
|
404
|
+
if (typeof options !== "object" || options === null) fail("Crawl options must be an object.");
|
|
405
|
+
assertCrawlOptionKeys(options);
|
|
406
|
+
const raw = options;
|
|
407
|
+
const urls = readUrls(raw);
|
|
408
|
+
const globPatterns = readObject(raw, "globPatterns");
|
|
409
|
+
const silent = readBoolean(raw, "silent", false);
|
|
410
|
+
const logger = readObject(raw, "logger");
|
|
411
|
+
return {
|
|
412
|
+
urls,
|
|
413
|
+
patterns: globPatterns && globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern),
|
|
414
|
+
output: resolve(normalize(readString(raw, "output") ?? CRAWL_DEFAULTS.output)),
|
|
415
|
+
depth: readDepth(raw),
|
|
416
|
+
maxPages: readMaxPages(raw),
|
|
417
|
+
sitemap: readSitemap(raw),
|
|
418
|
+
artifacts: readArtifacts(raw),
|
|
419
|
+
siteName: readString(raw, "siteName"),
|
|
420
|
+
description: readString(raw, "description"),
|
|
421
|
+
origin: readString(raw, "origin"),
|
|
422
|
+
driver: readDriver(raw),
|
|
423
|
+
useChrome: readBoolean(raw, "useChrome", false),
|
|
424
|
+
exclude: readExclude(raw),
|
|
425
|
+
crawlDelay: readCrawlDelay(raw),
|
|
426
|
+
skipSitemap: readBoolean(raw, "skipSitemap", false),
|
|
427
|
+
allowSubdomains: readBoolean(raw, "allowSubdomains", false),
|
|
428
|
+
stripBoilerplate: readBoolean(raw, "stripBoilerplate", true),
|
|
429
|
+
boilerplateThreshold: readBoilerplateThreshold(raw),
|
|
430
|
+
verbose: readBoolean(raw, "verbose", false),
|
|
431
|
+
logger: resolveLogger({
|
|
432
|
+
silent,
|
|
433
|
+
logger
|
|
434
|
+
}),
|
|
435
|
+
hooks: readObject(raw, "hooks")
|
|
436
|
+
};
|
|
437
|
+
}
|
|
438
|
+
function mergeCrawlOptions(...layers) {
|
|
439
|
+
const merged = {};
|
|
440
|
+
const exclude = [];
|
|
441
|
+
for (const layer of layers) for (const key in layer) {
|
|
442
|
+
const value = layer[key];
|
|
443
|
+
if (value === void 0) continue;
|
|
444
|
+
if (key === "exclude" && Array.isArray(value)) exclude.push(...value);
|
|
445
|
+
else merged[key] = value;
|
|
446
|
+
}
|
|
447
|
+
if (exclude.length > 0) merged.exclude = exclude;
|
|
448
|
+
return merged;
|
|
449
|
+
}
|
|
239
450
|
const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
|
|
240
451
|
const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
|
|
241
452
|
const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
|
|
@@ -244,6 +455,34 @@ const ROBOTS_CRAWL_DELAY_RE = /Crawl-delay:\s*(\d+(?:\.\d+)?)/i;
|
|
|
244
455
|
const URL_TRAILING_SLASH_RE = /\/$/;
|
|
245
456
|
const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
|
|
246
457
|
const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
|
|
458
|
+
const WINDOWS_DEVICE_NAMES = /* @__PURE__ */ new Set([
|
|
459
|
+
"con",
|
|
460
|
+
"prn",
|
|
461
|
+
"aux",
|
|
462
|
+
"nul",
|
|
463
|
+
"com1",
|
|
464
|
+
"com2",
|
|
465
|
+
"com3",
|
|
466
|
+
"com4",
|
|
467
|
+
"com5",
|
|
468
|
+
"com6",
|
|
469
|
+
"com7",
|
|
470
|
+
"com8",
|
|
471
|
+
"com9",
|
|
472
|
+
"lpt1",
|
|
473
|
+
"lpt2",
|
|
474
|
+
"lpt3",
|
|
475
|
+
"lpt4",
|
|
476
|
+
"lpt5",
|
|
477
|
+
"lpt6",
|
|
478
|
+
"lpt7",
|
|
479
|
+
"lpt8",
|
|
480
|
+
"lpt9"
|
|
481
|
+
]);
|
|
482
|
+
function sanitizeOutputSegment(segment) {
|
|
483
|
+
const safe = segment.replace(URL_PATH_UNSAFE_CHARS_RE, "-");
|
|
484
|
+
return safe.length >= 3 && safe.length <= 4 && WINDOWS_DEVICE_NAMES.has(safe.toLowerCase()) ? `~${safe}` : safe;
|
|
485
|
+
}
|
|
247
486
|
const TRACKING_PARAMS = /* @__PURE__ */ new Set([
|
|
248
487
|
"utm_source",
|
|
249
488
|
"utm_medium",
|
|
@@ -420,20 +659,17 @@ async function runConcurrent(items, concurrency, fn) {
|
|
|
420
659
|
await Promise.all(workers);
|
|
421
660
|
}
|
|
422
661
|
async function crawlAndGenerate(options, onProgress) {
|
|
423
|
-
|
|
424
|
-
|
|
662
|
+
return runCrawl(resolveCrawlOptions(options), onProgress);
|
|
663
|
+
}
|
|
664
|
+
async function runCrawl(options, onProgress) {
|
|
665
|
+
const { urls, patterns, output: outputDir, depth: maxDepth, maxPages, artifacts, origin, driver, useChrome, exclude, siteName: siteNameOption, description: descriptionOption, verbose, skipSitemap, sitemap: explicitSitemapUrls, allowSubdomains, stripBoilerplate, boilerplateThreshold, hooks: hooksConfig, logger } = options;
|
|
666
|
+
const writeLlmsTxt = artifacts.includes("llms.txt");
|
|
667
|
+
const writeLlmsFullTxt = artifacts.includes("llms-full.txt");
|
|
668
|
+
const writeMarkdown = artifacts.includes("markdown");
|
|
425
669
|
const hooks = createHooks();
|
|
426
670
|
if (hooksConfig) hooks.addHooks(hooksConfig);
|
|
427
|
-
if (onPage) hooks.hook("crawl:page", onPage);
|
|
428
671
|
const singlePageMode = maxDepth === 0;
|
|
429
|
-
|
|
430
|
-
let crawlDelay = userCrawlDelay;
|
|
431
|
-
let patterns;
|
|
432
|
-
try {
|
|
433
|
-
patterns = globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern);
|
|
434
|
-
} catch (error) {
|
|
435
|
-
throw new Error(`Invalid URL pattern: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
436
|
-
}
|
|
672
|
+
let crawlDelay = options.crawlDelay;
|
|
437
673
|
let startingUrls = patterns.map(getStartingUrl);
|
|
438
674
|
const hasGlobPatterns = patterns.some((p) => p.isGlob);
|
|
439
675
|
const progress = {
|
|
@@ -461,7 +697,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
461
697
|
if (startingUrls.length > 0 && !skipSitemap && !singlePageMode) {
|
|
462
698
|
const baseUrl = new URL(startingUrls[0]).origin;
|
|
463
699
|
const homePageUrl = baseUrl;
|
|
464
|
-
const hasExplicitSitemaps =
|
|
700
|
+
const hasExplicitSitemaps = explicitSitemapUrls.length > 0;
|
|
465
701
|
onProgress?.(progress);
|
|
466
702
|
let robotsContent = null;
|
|
467
703
|
try {
|
|
@@ -649,11 +885,11 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
649
885
|
title = pageData.title;
|
|
650
886
|
}
|
|
651
887
|
let filePath;
|
|
652
|
-
if (shouldProcessMarkdown &&
|
|
888
|
+
if (shouldProcessMarkdown && writeMarkdown) {
|
|
653
889
|
const urlPath = parsedUrl.pathname === "/" ? "/index" : parsedUrl.pathname;
|
|
654
|
-
const hostPrefix = allowSubdomains ? [parsedUrl.hostname
|
|
890
|
+
const hostPrefix = allowSubdomains ? [sanitizeOutputSegment(parsedUrl.hostname)] : [];
|
|
655
891
|
const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
|
|
656
|
-
const safeSegments = [...hostPrefix, ...pathSegments.map(
|
|
892
|
+
const safeSegments = [...hostPrefix, ...pathSegments.map(sanitizeOutputSegment)];
|
|
657
893
|
const filename = safeSegments.length > 0 ? safeSegments.join("/") : "index";
|
|
658
894
|
const safeFilename = normalize(`${filename}.md`);
|
|
659
895
|
filePath = join(outputDir, safeFilename);
|
|
@@ -674,7 +910,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
674
910
|
progress.crawling.processed = results.length;
|
|
675
911
|
onProgress?.(progress);
|
|
676
912
|
}
|
|
677
|
-
if (
|
|
913
|
+
if (!singlePageMode && !sitemapProvidedUrls && depth < maxDepth) {
|
|
678
914
|
const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link) && isContentUrl(link));
|
|
679
915
|
for (const link of filteredLinks) {
|
|
680
916
|
if (processedUrls.has(link)) continue;
|
|
@@ -693,7 +929,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
693
929
|
}
|
|
694
930
|
}
|
|
695
931
|
};
|
|
696
|
-
const urlsToProcess = startingUrls.slice(0,
|
|
932
|
+
const urlsToProcess = startingUrls.slice(0, maxPages);
|
|
697
933
|
for (const url of urlsToProcess) processedUrls.add(url.replace(URL_TRAILING_SLASH_RE, "") || url);
|
|
698
934
|
let totalProcessed = 0;
|
|
699
935
|
progress.crawling.status = "processing";
|
|
@@ -739,7 +975,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
739
975
|
depth: request.userData?.depth || 0
|
|
740
976
|
});
|
|
741
977
|
},
|
|
742
|
-
maxRequestsPerCrawl,
|
|
978
|
+
maxRequestsPerCrawl: Number.isFinite(maxPages) ? maxPages : void 0,
|
|
743
979
|
respectRobotsTxtFile: false
|
|
744
980
|
};
|
|
745
981
|
if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
|
|
@@ -755,8 +991,8 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
755
991
|
try {
|
|
756
992
|
await crawler.run(allRequests);
|
|
757
993
|
totalProcessed += urlsToProcess.length;
|
|
758
|
-
while (pendingUrls.length > 0 && totalProcessed <
|
|
759
|
-
const batch = pendingUrls.splice(0,
|
|
994
|
+
while (pendingUrls.length > 0 && totalProcessed < maxPages) {
|
|
995
|
+
const batch = pendingUrls.splice(0, maxPages - totalProcessed);
|
|
760
996
|
progress.crawling.total += batch.length;
|
|
761
997
|
onProgress?.(progress);
|
|
762
998
|
const batchRequests = batch.map((item) => ({
|
|
@@ -836,8 +1072,8 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
836
1072
|
};
|
|
837
1073
|
await runConcurrent(urlsToProcess, DEFAULT_CONCURRENCY, (url) => fetchPage(url, 0));
|
|
838
1074
|
totalProcessed += urlsToProcess.length;
|
|
839
|
-
while (pendingUrls.length > 0 && totalProcessed <
|
|
840
|
-
const batch = pendingUrls.splice(0,
|
|
1075
|
+
while (pendingUrls.length > 0 && totalProcessed < maxPages) {
|
|
1076
|
+
const batch = pendingUrls.splice(0, maxPages - totalProcessed);
|
|
841
1077
|
progress.crawling.total += batch.length;
|
|
842
1078
|
onProgress?.(progress);
|
|
843
1079
|
await runConcurrent(batch, DEFAULT_CONCURRENCY, (item) => fetchPage(item.url, item.depth));
|
|
@@ -852,10 +1088,10 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
852
1088
|
onProgress?.(progress);
|
|
853
1089
|
const successfulResults = results.filter((r) => r.success);
|
|
854
1090
|
if (stripBoilerplate && !singlePageMode) {
|
|
855
|
-
const cleaned = stripBoilerplateFromCorpus(successfulResults.map((r) => r.content),
|
|
1091
|
+
const cleaned = stripBoilerplateFromCorpus(successfulResults.map((r) => r.content), { threshold: boilerplateThreshold });
|
|
856
1092
|
for (let i = 0; i < successfulResults.length; i++) successfulResults[i].content = cleaned[i];
|
|
857
1093
|
}
|
|
858
|
-
if (
|
|
1094
|
+
if (writeMarkdown) for (const result of successfulResults) {
|
|
859
1095
|
if (!result.filePath) continue;
|
|
860
1096
|
const contentCtx = {
|
|
861
1097
|
url: result.url,
|
|
@@ -879,9 +1115,9 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
879
1115
|
const resultUrl = new URL(withHttps(r.url));
|
|
880
1116
|
return resultUrl.href === originUrl || resultUrl.href === `${originUrl}/`;
|
|
881
1117
|
});
|
|
882
|
-
const siteName =
|
|
883
|
-
const description =
|
|
884
|
-
if (
|
|
1118
|
+
const siteName = siteNameOption || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
|
|
1119
|
+
const description = descriptionOption || homePageResult?.metadata?.description || successfulResults[0]?.metadata?.description;
|
|
1120
|
+
if (writeLlmsTxt || writeLlmsFullTxt) {
|
|
885
1121
|
progress.generation.current = "Generating llms.txt files";
|
|
886
1122
|
onProgress?.(progress);
|
|
887
1123
|
const contentResults = successfulResults.filter((result) => {
|
|
@@ -905,15 +1141,15 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
905
1141
|
siteName,
|
|
906
1142
|
description,
|
|
907
1143
|
origin: originUrl || firstUrl.origin,
|
|
908
|
-
generateFull:
|
|
1144
|
+
generateFull: writeLlmsFullTxt,
|
|
909
1145
|
outputDir
|
|
910
1146
|
});
|
|
911
|
-
if (
|
|
1147
|
+
if (writeLlmsTxt) {
|
|
912
1148
|
progress.generation.current = "Writing llms.txt";
|
|
913
1149
|
onProgress?.(progress);
|
|
914
1150
|
await writeFile(join(outputDir, "llms.txt"), llmsResult.llmsTxt, "utf-8");
|
|
915
1151
|
}
|
|
916
|
-
if (
|
|
1152
|
+
if (writeLlmsFullTxt && llmsResult.llmsFullTxt) {
|
|
917
1153
|
progress.generation.current = "Writing llms-full.txt";
|
|
918
1154
|
onProgress?.(progress);
|
|
919
1155
|
await writeFile(join(outputDir, "llms-full.txt"), llmsResult.llmsFullTxt, "utf-8");
|
|
@@ -924,4 +1160,4 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
924
1160
|
}
|
|
925
1161
|
return results;
|
|
926
1162
|
}
|
|
927
|
-
export {
|
|
1163
|
+
export { CRAWL_ARTIFACTS, CRAWL_DEFAULTS, assertCrawlOptionKeys, crawlAndGenerate, createClackLogger, mergeCrawlOptions, resolveCrawlOptions, resolveLogger, runCrawl, silentLogger, stripBoilerplateFromCorpus, validateCrawlUrl };
|