@mdream/crawl 1.7.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -236,6 +236,217 @@ function resolveLogger(options) {
236
236
  if (options.logger) return options.logger;
237
237
  return options.silent ? silentLogger : createClackLogger();
238
238
  }
239
+ const CRAWL_ARTIFACTS = [
240
+ "llms.txt",
241
+ "llms-full.txt",
242
+ "markdown"
243
+ ];
244
+ const CRAWL_DRIVERS = ["http", "playwright"];
245
+ const CRAWL_DEFAULTS = {
246
+ output: "output",
247
+ depth: 3,
248
+ driver: "http",
249
+ artifacts: CRAWL_ARTIFACTS,
250
+ boilerplateThreshold: DEFAULT_BOILERPLATE_THRESHOLD
251
+ };
252
+ const OPTION_KEYS = {
253
+ urls: true,
254
+ output: true,
255
+ depth: true,
256
+ maxPages: true,
257
+ sitemap: true,
258
+ artifacts: true,
259
+ siteName: true,
260
+ description: true,
261
+ origin: true,
262
+ driver: true,
263
+ useChrome: true,
264
+ exclude: true,
265
+ crawlDelay: true,
266
+ skipSitemap: true,
267
+ allowSubdomains: true,
268
+ stripBoilerplate: true,
269
+ boilerplateThreshold: true,
270
+ verbose: true,
271
+ silent: true,
272
+ logger: true,
273
+ hooks: true,
274
+ globPatterns: true
275
+ };
276
+ const ARTIFACTS_HINT = "Use \"artifacts\" instead, for example artifacts: ['llms.txt', 'markdown'].";
277
+ const REMOVED_OPTIONS = {
278
+ maxRequestsPerCrawl: "Use \"maxPages\" instead.",
279
+ maxDepth: "Use \"depth\" instead.",
280
+ followLinks: "Use \"depth\" instead. Set depth to 0 to process only the given URLs.",
281
+ sitemapUrls: "Use \"sitemap\" instead.",
282
+ generateLlmsTxt: ARTIFACTS_HINT,
283
+ generateLlmsFullTxt: ARTIFACTS_HINT,
284
+ generateIndividualMd: ARTIFACTS_HINT,
285
+ siteNameOverride: "Use \"siteName\" instead.",
286
+ descriptionOverride: "Use \"description\" instead.",
287
+ outputDir: "Use \"output\" instead.",
288
+ chunkSize: "Remove it. It had no effect.",
289
+ onPage: "Use hooks: { 'crawl:page': fn } instead."
290
+ };
291
+ function fail(message) {
292
+ throw new TypeError(message);
293
+ }
294
+ function show(value) {
295
+ return typeof value === "string" ? JSON.stringify(value) : String(value);
296
+ }
297
+ function assertCrawlOptionKeys(options) {
298
+ const lines = [];
299
+ let hasTypo = false;
300
+ for (const key in options) {
301
+ if (options[key] === void 0 || key in OPTION_KEYS) continue;
302
+ const hint = REMOVED_OPTIONS[key];
303
+ if (hint === void 0) hasTypo = true;
304
+ lines.push(hint === void 0 ? `Unknown crawl option ${show(key)}.` : `Unknown crawl option ${show(key)}. ${hint}`);
305
+ }
306
+ if (lines.length === 0) return;
307
+ if (hasTypo) lines.push(`Valid options: ${Object.keys(OPTION_KEYS).join(", ")}.`);
308
+ fail(lines.join("\n"));
309
+ }
310
+ function validateCrawlUrl(url) {
311
+ const globError = validateGlobPattern(url);
312
+ if (globError) return globError;
313
+ if (!parseUrlPattern(url).isGlob && !URL.canParse(withHttps(url))) return `Invalid URL: ${show(url)}.`;
314
+ }
315
+ function readBoolean(raw, key, fallback) {
316
+ const value = raw[key];
317
+ if (value === void 0) return fallback;
318
+ if (typeof value !== "boolean") fail(`"${key}" must be true or false. You gave ${show(value)}.`);
319
+ return value;
320
+ }
321
+ function readString(raw, key) {
322
+ const value = raw[key];
323
+ if (value !== void 0 && typeof value !== "string") fail(`"${key}" must be a string. You gave ${show(value)}.`);
324
+ return value;
325
+ }
326
+ function readStrings(raw, key) {
327
+ const value = raw[key];
328
+ if (value === void 0) return void 0;
329
+ if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) fail(`"${key}" must be an array of strings.`);
330
+ return value;
331
+ }
332
+ function readObject(raw, key) {
333
+ const value = raw[key];
334
+ if (value !== void 0 && (typeof value !== "object" || value === null)) fail(`"${key}" must be an object.`);
335
+ return value;
336
+ }
337
+ function readUrls(raw) {
338
+ const urls = readStrings(raw, "urls");
339
+ if (!urls || urls.length === 0) fail("\"urls\" must be a non-empty array of URLs.");
340
+ for (const url of urls) {
341
+ const error = validateCrawlUrl(url);
342
+ if (error) fail(error);
343
+ }
344
+ return urls;
345
+ }
346
+ function readSitemap(raw) {
347
+ const value = raw.sitemap;
348
+ if (value === void 0) return [];
349
+ const list = typeof value === "string" ? [value] : value;
350
+ if (!Array.isArray(list) || !list.every((item) => typeof item === "string")) fail("\"sitemap\" must be a URL or an array of URLs.");
351
+ return list.map((item) => {
352
+ const url = withHttps(item);
353
+ if (!URL.canParse(url)) fail(`"sitemap" has an invalid URL: ${show(item)}.`);
354
+ return url;
355
+ });
356
+ }
357
+ function readArtifacts(raw) {
358
+ const value = raw.artifacts;
359
+ if (value === void 0) return [...CRAWL_DEFAULTS.artifacts];
360
+ const valid = `Valid artifacts: ${CRAWL_ARTIFACTS.join(", ")}.`;
361
+ if (!Array.isArray(value)) fail(`"artifacts" must be an array. ${valid}`);
362
+ for (const artifact of value) if (!CRAWL_ARTIFACTS.includes(artifact)) fail(`Invalid artifact ${show(artifact)}. ${valid}`);
363
+ return CRAWL_ARTIFACTS.filter((artifact) => value.includes(artifact));
364
+ }
365
+ function readDriver(raw) {
366
+ const value = raw.driver;
367
+ if (value === void 0) return CRAWL_DEFAULTS.driver;
368
+ if (!CRAWL_DRIVERS.includes(value)) fail(`"driver" must be "http" or "playwright". You gave ${show(value)}.`);
369
+ return value;
370
+ }
371
+ function readDepth(raw) {
372
+ const value = raw.depth;
373
+ if (value === void 0) return CRAWL_DEFAULTS.depth;
374
+ if (!Number.isInteger(value) || value < 0 || value > 10) fail(`"depth" must be an integer from 0 to 10. You gave ${show(value)}.`);
375
+ return value;
376
+ }
377
+ function readMaxPages(raw) {
378
+ const value = raw.maxPages;
379
+ if (value === void 0) return Number.POSITIVE_INFINITY;
380
+ if (!Number.isInteger(value) || value < 1) fail(`"maxPages" must be an integer of 1 or more. You gave ${show(value)}.`);
381
+ return value;
382
+ }
383
+ function readCrawlDelay(raw) {
384
+ const value = raw.crawlDelay;
385
+ if (value === void 0) return void 0;
386
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) fail(`"crawlDelay" must be a number of seconds, 0 or more. You gave ${show(value)}.`);
387
+ return value;
388
+ }
389
+ function readBoilerplateThreshold(raw) {
390
+ const value = raw.boilerplateThreshold;
391
+ if (value === void 0) return CRAWL_DEFAULTS.boilerplateThreshold;
392
+ if (typeof value !== "number" || !(value > 0 && value <= 1)) fail(`"boilerplateThreshold" must be a number greater than 0 and at most 1. You gave ${show(value)}.`);
393
+ return value;
394
+ }
395
+ function readExclude(raw) {
396
+ const exclude = readStrings(raw, "exclude") ?? [];
397
+ for (const pattern of exclude) {
398
+ const error = validateGlobPattern(pattern);
399
+ if (error) fail(`"exclude" has an invalid pattern. ${error}`);
400
+ }
401
+ return exclude;
402
+ }
403
+ function resolveCrawlOptions(options) {
404
+ if (typeof options !== "object" || options === null) fail("Crawl options must be an object.");
405
+ assertCrawlOptionKeys(options);
406
+ const raw = options;
407
+ const urls = readUrls(raw);
408
+ const globPatterns = readObject(raw, "globPatterns");
409
+ const silent = readBoolean(raw, "silent", false);
410
+ const logger = readObject(raw, "logger");
411
+ return {
412
+ urls,
413
+ patterns: globPatterns && globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern),
414
+ output: resolve(normalize(readString(raw, "output") ?? CRAWL_DEFAULTS.output)),
415
+ depth: readDepth(raw),
416
+ maxPages: readMaxPages(raw),
417
+ sitemap: readSitemap(raw),
418
+ artifacts: readArtifacts(raw),
419
+ siteName: readString(raw, "siteName"),
420
+ description: readString(raw, "description"),
421
+ origin: readString(raw, "origin"),
422
+ driver: readDriver(raw),
423
+ useChrome: readBoolean(raw, "useChrome", false),
424
+ exclude: readExclude(raw),
425
+ crawlDelay: readCrawlDelay(raw),
426
+ skipSitemap: readBoolean(raw, "skipSitemap", false),
427
+ allowSubdomains: readBoolean(raw, "allowSubdomains", false),
428
+ stripBoilerplate: readBoolean(raw, "stripBoilerplate", true),
429
+ boilerplateThreshold: readBoilerplateThreshold(raw),
430
+ verbose: readBoolean(raw, "verbose", false),
431
+ logger: resolveLogger({
432
+ silent,
433
+ logger
434
+ }),
435
+ hooks: readObject(raw, "hooks")
436
+ };
437
+ }
438
+ function mergeCrawlOptions(...layers) {
439
+ const merged = {};
440
+ const exclude = [];
441
+ for (const layer of layers) for (const key in layer) {
442
+ const value = layer[key];
443
+ if (value === void 0) continue;
444
+ if (key === "exclude" && Array.isArray(value)) exclude.push(...value);
445
+ else merged[key] = value;
446
+ }
447
+ if (exclude.length > 0) merged.exclude = exclude;
448
+ return merged;
449
+ }
239
450
  const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
240
451
  const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
241
452
  const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
@@ -420,20 +631,17 @@ async function runConcurrent(items, concurrency, fn) {
420
631
  await Promise.all(workers);
421
632
  }
422
633
  async function crawlAndGenerate(options, onProgress) {
423
- const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay: userCrawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, sitemapUrls: explicitSitemapUrls, allowSubdomains = false, stripBoilerplate = true, boilerplateThreshold, hooks: hooksConfig, onPage } = options;
424
- const logger = resolveLogger(options);
634
+ return runCrawl(resolveCrawlOptions(options), onProgress);
635
+ }
636
+ async function runCrawl(options, onProgress) {
637
+ const { urls, patterns, output: outputDir, depth: maxDepth, maxPages, artifacts, origin, driver, useChrome, exclude, siteName: siteNameOption, description: descriptionOption, verbose, skipSitemap, sitemap: explicitSitemapUrls, allowSubdomains, stripBoilerplate, boilerplateThreshold, hooks: hooksConfig, logger } = options;
638
+ const writeLlmsTxt = artifacts.includes("llms.txt");
639
+ const writeLlmsFullTxt = artifacts.includes("llms-full.txt");
640
+ const writeMarkdown = artifacts.includes("markdown");
425
641
  const hooks = createHooks();
426
642
  if (hooksConfig) hooks.addHooks(hooksConfig);
427
- if (onPage) hooks.hook("crawl:page", onPage);
428
643
  const singlePageMode = maxDepth === 0;
429
- const outputDir = resolve(normalize(rawOutputDir));
430
- let crawlDelay = userCrawlDelay;
431
- let patterns;
432
- try {
433
- patterns = globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern);
434
- } catch (error) {
435
- throw new Error(`Invalid URL pattern: ${error instanceof Error ? error.message : "Unknown error"}`);
436
- }
644
+ let crawlDelay = options.crawlDelay;
437
645
  let startingUrls = patterns.map(getStartingUrl);
438
646
  const hasGlobPatterns = patterns.some((p) => p.isGlob);
439
647
  const progress = {
@@ -461,7 +669,7 @@ async function crawlAndGenerate(options, onProgress) {
461
669
  if (startingUrls.length > 0 && !skipSitemap && !singlePageMode) {
462
670
  const baseUrl = new URL(startingUrls[0]).origin;
463
671
  const homePageUrl = baseUrl;
464
- const hasExplicitSitemaps = !!explicitSitemapUrls && explicitSitemapUrls.length > 0;
672
+ const hasExplicitSitemaps = explicitSitemapUrls.length > 0;
465
673
  onProgress?.(progress);
466
674
  let robotsContent = null;
467
675
  try {
@@ -649,7 +857,7 @@ async function crawlAndGenerate(options, onProgress) {
649
857
  title = pageData.title;
650
858
  }
651
859
  let filePath;
652
- if (shouldProcessMarkdown && generateIndividualMd) {
860
+ if (shouldProcessMarkdown && writeMarkdown) {
653
861
  const urlPath = parsedUrl.pathname === "/" ? "/index" : parsedUrl.pathname;
654
862
  const hostPrefix = allowSubdomains ? [parsedUrl.hostname.replace(URL_PATH_UNSAFE_CHARS_RE, "-")] : [];
655
863
  const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
@@ -674,7 +882,7 @@ async function crawlAndGenerate(options, onProgress) {
674
882
  progress.crawling.processed = results.length;
675
883
  onProgress?.(progress);
676
884
  }
677
- if (followLinks && !singlePageMode && !sitemapProvidedUrls && depth < maxDepth) {
885
+ if (!singlePageMode && !sitemapProvidedUrls && depth < maxDepth) {
678
886
  const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link) && isContentUrl(link));
679
887
  for (const link of filteredLinks) {
680
888
  if (processedUrls.has(link)) continue;
@@ -693,7 +901,7 @@ async function crawlAndGenerate(options, onProgress) {
693
901
  }
694
902
  }
695
903
  };
696
- const urlsToProcess = startingUrls.slice(0, maxRequestsPerCrawl);
904
+ const urlsToProcess = startingUrls.slice(0, maxPages);
697
905
  for (const url of urlsToProcess) processedUrls.add(url.replace(URL_TRAILING_SLASH_RE, "") || url);
698
906
  let totalProcessed = 0;
699
907
  progress.crawling.status = "processing";
@@ -739,7 +947,7 @@ async function crawlAndGenerate(options, onProgress) {
739
947
  depth: request.userData?.depth || 0
740
948
  });
741
949
  },
742
- maxRequestsPerCrawl,
950
+ maxRequestsPerCrawl: Number.isFinite(maxPages) ? maxPages : void 0,
743
951
  respectRobotsTxtFile: false
744
952
  };
745
953
  if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
@@ -755,8 +963,8 @@ async function crawlAndGenerate(options, onProgress) {
755
963
  try {
756
964
  await crawler.run(allRequests);
757
965
  totalProcessed += urlsToProcess.length;
758
- while (pendingUrls.length > 0 && totalProcessed < maxRequestsPerCrawl) {
759
- const batch = pendingUrls.splice(0, maxRequestsPerCrawl - totalProcessed);
966
+ while (pendingUrls.length > 0 && totalProcessed < maxPages) {
967
+ const batch = pendingUrls.splice(0, maxPages - totalProcessed);
760
968
  progress.crawling.total += batch.length;
761
969
  onProgress?.(progress);
762
970
  const batchRequests = batch.map((item) => ({
@@ -836,8 +1044,8 @@ async function crawlAndGenerate(options, onProgress) {
836
1044
  };
837
1045
  await runConcurrent(urlsToProcess, DEFAULT_CONCURRENCY, (url) => fetchPage(url, 0));
838
1046
  totalProcessed += urlsToProcess.length;
839
- while (pendingUrls.length > 0 && totalProcessed < maxRequestsPerCrawl) {
840
- const batch = pendingUrls.splice(0, maxRequestsPerCrawl - totalProcessed);
1047
+ while (pendingUrls.length > 0 && totalProcessed < maxPages) {
1048
+ const batch = pendingUrls.splice(0, maxPages - totalProcessed);
841
1049
  progress.crawling.total += batch.length;
842
1050
  onProgress?.(progress);
843
1051
  await runConcurrent(batch, DEFAULT_CONCURRENCY, (item) => fetchPage(item.url, item.depth));
@@ -852,10 +1060,10 @@ async function crawlAndGenerate(options, onProgress) {
852
1060
  onProgress?.(progress);
853
1061
  const successfulResults = results.filter((r) => r.success);
854
1062
  if (stripBoilerplate && !singlePageMode) {
855
- const cleaned = stripBoilerplateFromCorpus(successfulResults.map((r) => r.content), boilerplateThreshold !== void 0 ? { threshold: boilerplateThreshold } : {});
1063
+ const cleaned = stripBoilerplateFromCorpus(successfulResults.map((r) => r.content), { threshold: boilerplateThreshold });
856
1064
  for (let i = 0; i < successfulResults.length; i++) successfulResults[i].content = cleaned[i];
857
1065
  }
858
- if (generateIndividualMd) for (const result of successfulResults) {
1066
+ if (writeMarkdown) for (const result of successfulResults) {
859
1067
  if (!result.filePath) continue;
860
1068
  const contentCtx = {
861
1069
  url: result.url,
@@ -879,9 +1087,9 @@ async function crawlAndGenerate(options, onProgress) {
879
1087
  const resultUrl = new URL(withHttps(r.url));
880
1088
  return resultUrl.href === originUrl || resultUrl.href === `${originUrl}/`;
881
1089
  });
882
- const siteName = siteNameOverride || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
883
- const description = descriptionOverride || homePageResult?.metadata?.description || successfulResults[0]?.metadata?.description;
884
- if (generateLlmsTxt || generateLlmsFullTxt) {
1090
+ const siteName = siteNameOption || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
1091
+ const description = descriptionOption || homePageResult?.metadata?.description || successfulResults[0]?.metadata?.description;
1092
+ if (writeLlmsTxt || writeLlmsFullTxt) {
885
1093
  progress.generation.current = "Generating llms.txt files";
886
1094
  onProgress?.(progress);
887
1095
  const contentResults = successfulResults.filter((result) => {
@@ -905,15 +1113,15 @@ async function crawlAndGenerate(options, onProgress) {
905
1113
  siteName,
906
1114
  description,
907
1115
  origin: originUrl || firstUrl.origin,
908
- generateFull: generateLlmsFullTxt,
1116
+ generateFull: writeLlmsFullTxt,
909
1117
  outputDir
910
1118
  });
911
- if (generateLlmsTxt) {
1119
+ if (writeLlmsTxt) {
912
1120
  progress.generation.current = "Writing llms.txt";
913
1121
  onProgress?.(progress);
914
1122
  await writeFile(join(outputDir, "llms.txt"), llmsResult.llmsTxt, "utf-8");
915
1123
  }
916
- if (generateLlmsFullTxt && llmsResult.llmsFullTxt) {
1124
+ if (writeLlmsFullTxt && llmsResult.llmsFullTxt) {
917
1125
  progress.generation.current = "Writing llms-full.txt";
918
1126
  onProgress?.(progress);
919
1127
  await writeFile(join(outputDir, "llms-full.txt"), llmsResult.llmsFullTxt, "utf-8");
@@ -924,4 +1132,4 @@ async function crawlAndGenerate(options, onProgress) {
924
1132
  }
925
1133
  return results;
926
1134
  }
927
- export { DEFAULT_BOILERPLATE_THRESHOLD, crawlAndGenerate, createClackLogger, parseUrlPattern, resolveLogger, silentLogger, stripBoilerplateFromCorpus, validateGlobPattern };
1135
+ export { CRAWL_ARTIFACTS, CRAWL_DEFAULTS, assertCrawlOptionKeys, crawlAndGenerate, createClackLogger, mergeCrawlOptions, resolveCrawlOptions, resolveLogger, runCrawl, silentLogger, stripBoilerplateFromCorpus, validateCrawlUrl };