@mdream/crawl 2.0.0-beta.1 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,7 +5,7 @@ import { createHooks } from "hookable";
5
5
  import { htmlToMarkdown } from "mdream";
6
6
  import { ofetch } from "ofetch";
7
7
  import { dirname, join, normalize, resolve } from "pathe";
8
- import { withHttps } from "ufo";
8
+ import { hasProtocol, withHttps } from "ufo";
9
9
  import picomatch from "picomatch";
10
10
  import { getDomain } from "tldts";
11
11
  import * as p from "@clack/prompts";
@@ -164,14 +164,14 @@ function matchesGlobPattern(url, parsedPattern, allowSubdomains = false) {
164
164
  }
165
165
  }
166
166
  function getStartingUrl(parsedPattern) {
167
- if (!parsedPattern.isGlob) return withHttps(parsedPattern.baseUrl);
167
+ const baseUrl = hasProtocol(parsedPattern.baseUrl) ? parsedPattern.baseUrl : withHttps(parsedPattern.baseUrl);
168
+ if (!parsedPattern.isGlob) return baseUrl;
168
169
  const pattern = parsedPattern.pattern;
169
170
  const firstGlobIndex = pattern.search(GLOB_CHAR_RE);
170
- if (firstGlobIndex === -1) return withHttps(parsedPattern.baseUrl + pattern);
171
+ if (firstGlobIndex === -1) return baseUrl + pattern;
171
172
  const beforeGlob = pattern.substring(0, firstGlobIndex);
172
173
  const lastSlash = beforeGlob.lastIndexOf("/");
173
- const pathBeforeGlob = lastSlash >= 0 ? beforeGlob.substring(0, lastSlash + 1) : "/";
174
- return withHttps(parsedPattern.baseUrl + pathBeforeGlob);
174
+ return baseUrl + (lastSlash >= 0 ? beforeGlob.substring(0, lastSlash + 1) : "/");
175
175
  }
176
176
  function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
177
177
  if (!excludePatterns || excludePatterns.length === 0) return false;
@@ -236,6 +236,217 @@ function resolveLogger(options) {
236
236
  if (options.logger) return options.logger;
237
237
  return options.silent ? silentLogger : createClackLogger();
238
238
  }
239
+ const CRAWL_ARTIFACTS = [
240
+ "llms.txt",
241
+ "llms-full.txt",
242
+ "markdown"
243
+ ];
244
+ const CRAWL_DRIVERS = ["http", "playwright"];
245
+ const CRAWL_DEFAULTS = {
246
+ output: "output",
247
+ depth: 3,
248
+ driver: "http",
249
+ artifacts: CRAWL_ARTIFACTS,
250
+ boilerplateThreshold: DEFAULT_BOILERPLATE_THRESHOLD
251
+ };
252
+ const OPTION_KEYS = {
253
+ urls: true,
254
+ output: true,
255
+ depth: true,
256
+ maxPages: true,
257
+ sitemap: true,
258
+ artifacts: true,
259
+ siteName: true,
260
+ description: true,
261
+ origin: true,
262
+ driver: true,
263
+ useChrome: true,
264
+ exclude: true,
265
+ crawlDelay: true,
266
+ skipSitemap: true,
267
+ allowSubdomains: true,
268
+ stripBoilerplate: true,
269
+ boilerplateThreshold: true,
270
+ verbose: true,
271
+ silent: true,
272
+ logger: true,
273
+ hooks: true,
274
+ globPatterns: true
275
+ };
276
+ const ARTIFACTS_HINT = "Use \"artifacts\" instead, for example artifacts: ['llms.txt', 'markdown'].";
277
+ const REMOVED_OPTIONS = {
278
+ maxRequestsPerCrawl: "Use \"maxPages\" instead.",
279
+ maxDepth: "Use \"depth\" instead.",
280
+ followLinks: "Use \"depth\" instead. Set depth to 0 to process only the given URLs.",
281
+ sitemapUrls: "Use \"sitemap\" instead.",
282
+ generateLlmsTxt: ARTIFACTS_HINT,
283
+ generateLlmsFullTxt: ARTIFACTS_HINT,
284
+ generateIndividualMd: ARTIFACTS_HINT,
285
+ siteNameOverride: "Use \"siteName\" instead.",
286
+ descriptionOverride: "Use \"description\" instead.",
287
+ outputDir: "Use \"output\" instead.",
288
+ chunkSize: "Remove it. It had no effect.",
289
+ onPage: "Use hooks: { 'crawl:page': fn } instead."
290
+ };
291
+ function fail(message) {
292
+ throw new TypeError(message);
293
+ }
294
+ function show(value) {
295
+ return typeof value === "string" ? JSON.stringify(value) : String(value);
296
+ }
297
+ function assertCrawlOptionKeys(options) {
298
+ const lines = [];
299
+ let hasTypo = false;
300
+ for (const key in options) {
301
+ if (options[key] === void 0 || key in OPTION_KEYS) continue;
302
+ const hint = REMOVED_OPTIONS[key];
303
+ if (hint === void 0) hasTypo = true;
304
+ lines.push(hint === void 0 ? `Unknown crawl option ${show(key)}.` : `Unknown crawl option ${show(key)}. ${hint}`);
305
+ }
306
+ if (lines.length === 0) return;
307
+ if (hasTypo) lines.push(`Valid options: ${Object.keys(OPTION_KEYS).join(", ")}.`);
308
+ fail(lines.join("\n"));
309
+ }
310
+ function validateCrawlUrl(url) {
311
+ const globError = validateGlobPattern(url);
312
+ if (globError) return globError;
313
+ if (!parseUrlPattern(url).isGlob && !URL.canParse(withHttps(url))) return `Invalid URL: ${show(url)}.`;
314
+ }
315
+ function readBoolean(raw, key, fallback) {
316
+ const value = raw[key];
317
+ if (value === void 0) return fallback;
318
+ if (typeof value !== "boolean") fail(`"${key}" must be true or false. You gave ${show(value)}.`);
319
+ return value;
320
+ }
321
+ function readString(raw, key) {
322
+ const value = raw[key];
323
+ if (value !== void 0 && typeof value !== "string") fail(`"${key}" must be a string. You gave ${show(value)}.`);
324
+ return value;
325
+ }
326
+ function readStrings(raw, key) {
327
+ const value = raw[key];
328
+ if (value === void 0) return void 0;
329
+ if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) fail(`"${key}" must be an array of strings.`);
330
+ return value;
331
+ }
332
+ function readObject(raw, key) {
333
+ const value = raw[key];
334
+ if (value !== void 0 && (typeof value !== "object" || value === null)) fail(`"${key}" must be an object.`);
335
+ return value;
336
+ }
337
+ function readUrls(raw) {
338
+ const urls = readStrings(raw, "urls");
339
+ if (!urls || urls.length === 0) fail("\"urls\" must be a non-empty array of URLs.");
340
+ for (const url of urls) {
341
+ const error = validateCrawlUrl(url);
342
+ if (error) fail(error);
343
+ }
344
+ return urls;
345
+ }
346
+ function readSitemap(raw) {
347
+ const value = raw.sitemap;
348
+ if (value === void 0) return [];
349
+ const list = typeof value === "string" ? [value] : value;
350
+ if (!Array.isArray(list) || !list.every((item) => typeof item === "string")) fail("\"sitemap\" must be a URL or an array of URLs.");
351
+ return list.map((item) => {
352
+ const url = withHttps(item);
353
+ if (!URL.canParse(url)) fail(`"sitemap" has an invalid URL: ${show(item)}.`);
354
+ return url;
355
+ });
356
+ }
357
+ function readArtifacts(raw) {
358
+ const value = raw.artifacts;
359
+ if (value === void 0) return [...CRAWL_DEFAULTS.artifacts];
360
+ const valid = `Valid artifacts: ${CRAWL_ARTIFACTS.join(", ")}.`;
361
+ if (!Array.isArray(value)) fail(`"artifacts" must be an array. ${valid}`);
362
+ for (const artifact of value) if (!CRAWL_ARTIFACTS.includes(artifact)) fail(`Invalid artifact ${show(artifact)}. ${valid}`);
363
+ return CRAWL_ARTIFACTS.filter((artifact) => value.includes(artifact));
364
+ }
365
+ function readDriver(raw) {
366
+ const value = raw.driver;
367
+ if (value === void 0) return CRAWL_DEFAULTS.driver;
368
+ if (!CRAWL_DRIVERS.includes(value)) fail(`"driver" must be "http" or "playwright". You gave ${show(value)}.`);
369
+ return value;
370
+ }
371
+ function readDepth(raw) {
372
+ const value = raw.depth;
373
+ if (value === void 0) return CRAWL_DEFAULTS.depth;
374
+ if (!Number.isInteger(value) || value < 0 || value > 10) fail(`"depth" must be an integer from 0 to 10. You gave ${show(value)}.`);
375
+ return value;
376
+ }
377
+ function readMaxPages(raw) {
378
+ const value = raw.maxPages;
379
+ if (value === void 0) return Number.POSITIVE_INFINITY;
380
+ if (!Number.isInteger(value) || value < 1) fail(`"maxPages" must be an integer of 1 or more. You gave ${show(value)}.`);
381
+ return value;
382
+ }
383
+ function readCrawlDelay(raw) {
384
+ const value = raw.crawlDelay;
385
+ if (value === void 0) return void 0;
386
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) fail(`"crawlDelay" must be a number of seconds, 0 or more. You gave ${show(value)}.`);
387
+ return value;
388
+ }
389
+ function readBoilerplateThreshold(raw) {
390
+ const value = raw.boilerplateThreshold;
391
+ if (value === void 0) return CRAWL_DEFAULTS.boilerplateThreshold;
392
+ if (typeof value !== "number" || !(value > 0 && value <= 1)) fail(`"boilerplateThreshold" must be a number greater than 0 and at most 1. You gave ${show(value)}.`);
393
+ return value;
394
+ }
395
+ function readExclude(raw) {
396
+ const exclude = readStrings(raw, "exclude") ?? [];
397
+ for (const pattern of exclude) {
398
+ const error = validateGlobPattern(pattern);
399
+ if (error) fail(`"exclude" has an invalid pattern. ${error}`);
400
+ }
401
+ return exclude;
402
+ }
403
+ function resolveCrawlOptions(options) {
404
+ if (typeof options !== "object" || options === null) fail("Crawl options must be an object.");
405
+ assertCrawlOptionKeys(options);
406
+ const raw = options;
407
+ const urls = readUrls(raw);
408
+ const globPatterns = readObject(raw, "globPatterns");
409
+ const silent = readBoolean(raw, "silent", false);
410
+ const logger = readObject(raw, "logger");
411
+ return {
412
+ urls,
413
+ patterns: globPatterns && globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern),
414
+ output: resolve(normalize(readString(raw, "output") ?? CRAWL_DEFAULTS.output)),
415
+ depth: readDepth(raw),
416
+ maxPages: readMaxPages(raw),
417
+ sitemap: readSitemap(raw),
418
+ artifacts: readArtifacts(raw),
419
+ siteName: readString(raw, "siteName"),
420
+ description: readString(raw, "description"),
421
+ origin: readString(raw, "origin"),
422
+ driver: readDriver(raw),
423
+ useChrome: readBoolean(raw, "useChrome", false),
424
+ exclude: readExclude(raw),
425
+ crawlDelay: readCrawlDelay(raw),
426
+ skipSitemap: readBoolean(raw, "skipSitemap", false),
427
+ allowSubdomains: readBoolean(raw, "allowSubdomains", false),
428
+ stripBoilerplate: readBoolean(raw, "stripBoilerplate", true),
429
+ boilerplateThreshold: readBoilerplateThreshold(raw),
430
+ verbose: readBoolean(raw, "verbose", false),
431
+ logger: resolveLogger({
432
+ silent,
433
+ logger
434
+ }),
435
+ hooks: readObject(raw, "hooks")
436
+ };
437
+ }
438
+ function mergeCrawlOptions(...layers) {
439
+ const merged = {};
440
+ const exclude = [];
441
+ for (const layer of layers) for (const key in layer) {
442
+ const value = layer[key];
443
+ if (value === void 0) continue;
444
+ if (key === "exclude" && Array.isArray(value)) exclude.push(...value);
445
+ else merged[key] = value;
446
+ }
447
+ if (exclude.length > 0) merged.exclude = exclude;
448
+ return merged;
449
+ }
239
450
  const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
240
451
  const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
241
452
  const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
@@ -244,6 +455,34 @@ const ROBOTS_CRAWL_DELAY_RE = /Crawl-delay:\s*(\d+(?:\.\d+)?)/i;
244
455
  const URL_TRAILING_SLASH_RE = /\/$/;
245
456
  const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
246
457
  const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
458
+ const WINDOWS_DEVICE_NAMES = /* @__PURE__ */ new Set([
459
+ "con",
460
+ "prn",
461
+ "aux",
462
+ "nul",
463
+ "com1",
464
+ "com2",
465
+ "com3",
466
+ "com4",
467
+ "com5",
468
+ "com6",
469
+ "com7",
470
+ "com8",
471
+ "com9",
472
+ "lpt1",
473
+ "lpt2",
474
+ "lpt3",
475
+ "lpt4",
476
+ "lpt5",
477
+ "lpt6",
478
+ "lpt7",
479
+ "lpt8",
480
+ "lpt9"
481
+ ]);
482
+ function sanitizeOutputSegment(segment) {
483
+ const safe = segment.replace(URL_PATH_UNSAFE_CHARS_RE, "-");
484
+ return safe.length >= 3 && safe.length <= 4 && WINDOWS_DEVICE_NAMES.has(safe.toLowerCase()) ? `~${safe}` : safe;
485
+ }
247
486
  const TRACKING_PARAMS = /* @__PURE__ */ new Set([
248
487
  "utm_source",
249
488
  "utm_medium",
@@ -420,20 +659,17 @@ async function runConcurrent(items, concurrency, fn) {
420
659
  await Promise.all(workers);
421
660
  }
422
661
  async function crawlAndGenerate(options, onProgress) {
423
- const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay: userCrawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, sitemapUrls: explicitSitemapUrls, allowSubdomains = false, stripBoilerplate = true, boilerplateThreshold, hooks: hooksConfig, onPage } = options;
424
- const logger = resolveLogger(options);
662
+ return runCrawl(resolveCrawlOptions(options), onProgress);
663
+ }
664
+ async function runCrawl(options, onProgress) {
665
+ const { urls, patterns, output: outputDir, depth: maxDepth, maxPages, artifacts, origin, driver, useChrome, exclude, siteName: siteNameOption, description: descriptionOption, verbose, skipSitemap, sitemap: explicitSitemapUrls, allowSubdomains, stripBoilerplate, boilerplateThreshold, hooks: hooksConfig, logger } = options;
666
+ const writeLlmsTxt = artifacts.includes("llms.txt");
667
+ const writeLlmsFullTxt = artifacts.includes("llms-full.txt");
668
+ const writeMarkdown = artifacts.includes("markdown");
425
669
  const hooks = createHooks();
426
670
  if (hooksConfig) hooks.addHooks(hooksConfig);
427
- if (onPage) hooks.hook("crawl:page", onPage);
428
671
  const singlePageMode = maxDepth === 0;
429
- const outputDir = resolve(normalize(rawOutputDir));
430
- let crawlDelay = userCrawlDelay;
431
- let patterns;
432
- try {
433
- patterns = globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern);
434
- } catch (error) {
435
- throw new Error(`Invalid URL pattern: ${error instanceof Error ? error.message : "Unknown error"}`);
436
- }
672
+ let crawlDelay = options.crawlDelay;
437
673
  let startingUrls = patterns.map(getStartingUrl);
438
674
  const hasGlobPatterns = patterns.some((p) => p.isGlob);
439
675
  const progress = {
@@ -461,7 +697,7 @@ async function crawlAndGenerate(options, onProgress) {
461
697
  if (startingUrls.length > 0 && !skipSitemap && !singlePageMode) {
462
698
  const baseUrl = new URL(startingUrls[0]).origin;
463
699
  const homePageUrl = baseUrl;
464
- const hasExplicitSitemaps = !!explicitSitemapUrls && explicitSitemapUrls.length > 0;
700
+ const hasExplicitSitemaps = explicitSitemapUrls.length > 0;
465
701
  onProgress?.(progress);
466
702
  let robotsContent = null;
467
703
  try {
@@ -649,11 +885,11 @@ async function crawlAndGenerate(options, onProgress) {
649
885
  title = pageData.title;
650
886
  }
651
887
  let filePath;
652
- if (shouldProcessMarkdown && generateIndividualMd) {
888
+ if (shouldProcessMarkdown && writeMarkdown) {
653
889
  const urlPath = parsedUrl.pathname === "/" ? "/index" : parsedUrl.pathname;
654
- const hostPrefix = allowSubdomains ? [parsedUrl.hostname.replace(URL_PATH_UNSAFE_CHARS_RE, "-")] : [];
890
+ const hostPrefix = allowSubdomains ? [sanitizeOutputSegment(parsedUrl.hostname)] : [];
655
891
  const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
656
- const safeSegments = [...hostPrefix, ...pathSegments.map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"))];
892
+ const safeSegments = [...hostPrefix, ...pathSegments.map(sanitizeOutputSegment)];
657
893
  const filename = safeSegments.length > 0 ? safeSegments.join("/") : "index";
658
894
  const safeFilename = normalize(`${filename}.md`);
659
895
  filePath = join(outputDir, safeFilename);
@@ -674,7 +910,7 @@ async function crawlAndGenerate(options, onProgress) {
674
910
  progress.crawling.processed = results.length;
675
911
  onProgress?.(progress);
676
912
  }
677
- if (followLinks && !singlePageMode && !sitemapProvidedUrls && depth < maxDepth) {
913
+ if (!singlePageMode && !sitemapProvidedUrls && depth < maxDepth) {
678
914
  const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link) && isContentUrl(link));
679
915
  for (const link of filteredLinks) {
680
916
  if (processedUrls.has(link)) continue;
@@ -693,7 +929,7 @@ async function crawlAndGenerate(options, onProgress) {
693
929
  }
694
930
  }
695
931
  };
696
- const urlsToProcess = startingUrls.slice(0, maxRequestsPerCrawl);
932
+ const urlsToProcess = startingUrls.slice(0, maxPages);
697
933
  for (const url of urlsToProcess) processedUrls.add(url.replace(URL_TRAILING_SLASH_RE, "") || url);
698
934
  let totalProcessed = 0;
699
935
  progress.crawling.status = "processing";
@@ -739,7 +975,7 @@ async function crawlAndGenerate(options, onProgress) {
739
975
  depth: request.userData?.depth || 0
740
976
  });
741
977
  },
742
- maxRequestsPerCrawl,
978
+ maxRequestsPerCrawl: Number.isFinite(maxPages) ? maxPages : void 0,
743
979
  respectRobotsTxtFile: false
744
980
  };
745
981
  if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
@@ -755,8 +991,8 @@ async function crawlAndGenerate(options, onProgress) {
755
991
  try {
756
992
  await crawler.run(allRequests);
757
993
  totalProcessed += urlsToProcess.length;
758
- while (pendingUrls.length > 0 && totalProcessed < maxRequestsPerCrawl) {
759
- const batch = pendingUrls.splice(0, maxRequestsPerCrawl - totalProcessed);
994
+ while (pendingUrls.length > 0 && totalProcessed < maxPages) {
995
+ const batch = pendingUrls.splice(0, maxPages - totalProcessed);
760
996
  progress.crawling.total += batch.length;
761
997
  onProgress?.(progress);
762
998
  const batchRequests = batch.map((item) => ({
@@ -836,8 +1072,8 @@ async function crawlAndGenerate(options, onProgress) {
836
1072
  };
837
1073
  await runConcurrent(urlsToProcess, DEFAULT_CONCURRENCY, (url) => fetchPage(url, 0));
838
1074
  totalProcessed += urlsToProcess.length;
839
- while (pendingUrls.length > 0 && totalProcessed < maxRequestsPerCrawl) {
840
- const batch = pendingUrls.splice(0, maxRequestsPerCrawl - totalProcessed);
1075
+ while (pendingUrls.length > 0 && totalProcessed < maxPages) {
1076
+ const batch = pendingUrls.splice(0, maxPages - totalProcessed);
841
1077
  progress.crawling.total += batch.length;
842
1078
  onProgress?.(progress);
843
1079
  await runConcurrent(batch, DEFAULT_CONCURRENCY, (item) => fetchPage(item.url, item.depth));
@@ -852,10 +1088,10 @@ async function crawlAndGenerate(options, onProgress) {
852
1088
  onProgress?.(progress);
853
1089
  const successfulResults = results.filter((r) => r.success);
854
1090
  if (stripBoilerplate && !singlePageMode) {
855
- const cleaned = stripBoilerplateFromCorpus(successfulResults.map((r) => r.content), boilerplateThreshold !== void 0 ? { threshold: boilerplateThreshold } : {});
1091
+ const cleaned = stripBoilerplateFromCorpus(successfulResults.map((r) => r.content), { threshold: boilerplateThreshold });
856
1092
  for (let i = 0; i < successfulResults.length; i++) successfulResults[i].content = cleaned[i];
857
1093
  }
858
- if (generateIndividualMd) for (const result of successfulResults) {
1094
+ if (writeMarkdown) for (const result of successfulResults) {
859
1095
  if (!result.filePath) continue;
860
1096
  const contentCtx = {
861
1097
  url: result.url,
@@ -879,9 +1115,9 @@ async function crawlAndGenerate(options, onProgress) {
879
1115
  const resultUrl = new URL(withHttps(r.url));
880
1116
  return resultUrl.href === originUrl || resultUrl.href === `${originUrl}/`;
881
1117
  });
882
- const siteName = siteNameOverride || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
883
- const description = descriptionOverride || homePageResult?.metadata?.description || successfulResults[0]?.metadata?.description;
884
- if (generateLlmsTxt || generateLlmsFullTxt) {
1118
+ const siteName = siteNameOption || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
1119
+ const description = descriptionOption || homePageResult?.metadata?.description || successfulResults[0]?.metadata?.description;
1120
+ if (writeLlmsTxt || writeLlmsFullTxt) {
885
1121
  progress.generation.current = "Generating llms.txt files";
886
1122
  onProgress?.(progress);
887
1123
  const contentResults = successfulResults.filter((result) => {
@@ -905,15 +1141,15 @@ async function crawlAndGenerate(options, onProgress) {
905
1141
  siteName,
906
1142
  description,
907
1143
  origin: originUrl || firstUrl.origin,
908
- generateFull: generateLlmsFullTxt,
1144
+ generateFull: writeLlmsFullTxt,
909
1145
  outputDir
910
1146
  });
911
- if (generateLlmsTxt) {
1147
+ if (writeLlmsTxt) {
912
1148
  progress.generation.current = "Writing llms.txt";
913
1149
  onProgress?.(progress);
914
1150
  await writeFile(join(outputDir, "llms.txt"), llmsResult.llmsTxt, "utf-8");
915
1151
  }
916
- if (generateLlmsFullTxt && llmsResult.llmsFullTxt) {
1152
+ if (writeLlmsFullTxt && llmsResult.llmsFullTxt) {
917
1153
  progress.generation.current = "Writing llms-full.txt";
918
1154
  onProgress?.(progress);
919
1155
  await writeFile(join(outputDir, "llms-full.txt"), llmsResult.llmsFullTxt, "utf-8");
@@ -924,4 +1160,4 @@ async function crawlAndGenerate(options, onProgress) {
924
1160
  }
925
1161
  return results;
926
1162
  }
927
- export { DEFAULT_BOILERPLATE_THRESHOLD, crawlAndGenerate, createClackLogger, parseUrlPattern, resolveLogger, silentLogger, stripBoilerplateFromCorpus, validateGlobPattern };
1163
+ export { CRAWL_ARTIFACTS, CRAWL_DEFAULTS, assertCrawlOptionKeys, crawlAndGenerate, createClackLogger, mergeCrawlOptions, resolveCrawlOptions, resolveLogger, runCrawl, silentLogger, stripBoilerplateFromCorpus, validateCrawlUrl };