@mdream/crawl 2.0.0-beta.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +194 -88
- package/dist/_chunks/crawl.mjs +237 -29
- package/dist/cli.mjs +328 -358
- package/dist/index.d.mts +64 -59
- package/package.json +14 -4
package/dist/_chunks/crawl.mjs
CHANGED
|
@@ -236,6 +236,217 @@ function resolveLogger(options) {
|
|
|
236
236
|
if (options.logger) return options.logger;
|
|
237
237
|
return options.silent ? silentLogger : createClackLogger();
|
|
238
238
|
}
|
|
239
|
+
const CRAWL_ARTIFACTS = [
|
|
240
|
+
"llms.txt",
|
|
241
|
+
"llms-full.txt",
|
|
242
|
+
"markdown"
|
|
243
|
+
];
|
|
244
|
+
const CRAWL_DRIVERS = ["http", "playwright"];
|
|
245
|
+
const CRAWL_DEFAULTS = {
|
|
246
|
+
output: "output",
|
|
247
|
+
depth: 3,
|
|
248
|
+
driver: "http",
|
|
249
|
+
artifacts: CRAWL_ARTIFACTS,
|
|
250
|
+
boilerplateThreshold: DEFAULT_BOILERPLATE_THRESHOLD
|
|
251
|
+
};
|
|
252
|
+
const OPTION_KEYS = {
|
|
253
|
+
urls: true,
|
|
254
|
+
output: true,
|
|
255
|
+
depth: true,
|
|
256
|
+
maxPages: true,
|
|
257
|
+
sitemap: true,
|
|
258
|
+
artifacts: true,
|
|
259
|
+
siteName: true,
|
|
260
|
+
description: true,
|
|
261
|
+
origin: true,
|
|
262
|
+
driver: true,
|
|
263
|
+
useChrome: true,
|
|
264
|
+
exclude: true,
|
|
265
|
+
crawlDelay: true,
|
|
266
|
+
skipSitemap: true,
|
|
267
|
+
allowSubdomains: true,
|
|
268
|
+
stripBoilerplate: true,
|
|
269
|
+
boilerplateThreshold: true,
|
|
270
|
+
verbose: true,
|
|
271
|
+
silent: true,
|
|
272
|
+
logger: true,
|
|
273
|
+
hooks: true,
|
|
274
|
+
globPatterns: true
|
|
275
|
+
};
|
|
276
|
+
const ARTIFACTS_HINT = "Use \"artifacts\" instead, for example artifacts: ['llms.txt', 'markdown'].";
|
|
277
|
+
const REMOVED_OPTIONS = {
|
|
278
|
+
maxRequestsPerCrawl: "Use \"maxPages\" instead.",
|
|
279
|
+
maxDepth: "Use \"depth\" instead.",
|
|
280
|
+
followLinks: "Use \"depth\" instead. Set depth to 0 to process only the given URLs.",
|
|
281
|
+
sitemapUrls: "Use \"sitemap\" instead.",
|
|
282
|
+
generateLlmsTxt: ARTIFACTS_HINT,
|
|
283
|
+
generateLlmsFullTxt: ARTIFACTS_HINT,
|
|
284
|
+
generateIndividualMd: ARTIFACTS_HINT,
|
|
285
|
+
siteNameOverride: "Use \"siteName\" instead.",
|
|
286
|
+
descriptionOverride: "Use \"description\" instead.",
|
|
287
|
+
outputDir: "Use \"output\" instead.",
|
|
288
|
+
chunkSize: "Remove it. It had no effect.",
|
|
289
|
+
onPage: "Use hooks: { 'crawl:page': fn } instead."
|
|
290
|
+
};
|
|
291
|
+
function fail(message) {
|
|
292
|
+
throw new TypeError(message);
|
|
293
|
+
}
|
|
294
|
+
function show(value) {
|
|
295
|
+
return typeof value === "string" ? JSON.stringify(value) : String(value);
|
|
296
|
+
}
|
|
297
|
+
function assertCrawlOptionKeys(options) {
|
|
298
|
+
const lines = [];
|
|
299
|
+
let hasTypo = false;
|
|
300
|
+
for (const key in options) {
|
|
301
|
+
if (options[key] === void 0 || key in OPTION_KEYS) continue;
|
|
302
|
+
const hint = REMOVED_OPTIONS[key];
|
|
303
|
+
if (hint === void 0) hasTypo = true;
|
|
304
|
+
lines.push(hint === void 0 ? `Unknown crawl option ${show(key)}.` : `Unknown crawl option ${show(key)}. ${hint}`);
|
|
305
|
+
}
|
|
306
|
+
if (lines.length === 0) return;
|
|
307
|
+
if (hasTypo) lines.push(`Valid options: ${Object.keys(OPTION_KEYS).join(", ")}.`);
|
|
308
|
+
fail(lines.join("\n"));
|
|
309
|
+
}
|
|
310
|
+
function validateCrawlUrl(url) {
|
|
311
|
+
const globError = validateGlobPattern(url);
|
|
312
|
+
if (globError) return globError;
|
|
313
|
+
if (!parseUrlPattern(url).isGlob && !URL.canParse(withHttps(url))) return `Invalid URL: ${show(url)}.`;
|
|
314
|
+
}
|
|
315
|
+
function readBoolean(raw, key, fallback) {
|
|
316
|
+
const value = raw[key];
|
|
317
|
+
if (value === void 0) return fallback;
|
|
318
|
+
if (typeof value !== "boolean") fail(`"${key}" must be true or false. You gave ${show(value)}.`);
|
|
319
|
+
return value;
|
|
320
|
+
}
|
|
321
|
+
function readString(raw, key) {
|
|
322
|
+
const value = raw[key];
|
|
323
|
+
if (value !== void 0 && typeof value !== "string") fail(`"${key}" must be a string. You gave ${show(value)}.`);
|
|
324
|
+
return value;
|
|
325
|
+
}
|
|
326
|
+
function readStrings(raw, key) {
|
|
327
|
+
const value = raw[key];
|
|
328
|
+
if (value === void 0) return void 0;
|
|
329
|
+
if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) fail(`"${key}" must be an array of strings.`);
|
|
330
|
+
return value;
|
|
331
|
+
}
|
|
332
|
+
function readObject(raw, key) {
|
|
333
|
+
const value = raw[key];
|
|
334
|
+
if (value !== void 0 && (typeof value !== "object" || value === null)) fail(`"${key}" must be an object.`);
|
|
335
|
+
return value;
|
|
336
|
+
}
|
|
337
|
+
function readUrls(raw) {
|
|
338
|
+
const urls = readStrings(raw, "urls");
|
|
339
|
+
if (!urls || urls.length === 0) fail("\"urls\" must be a non-empty array of URLs.");
|
|
340
|
+
for (const url of urls) {
|
|
341
|
+
const error = validateCrawlUrl(url);
|
|
342
|
+
if (error) fail(error);
|
|
343
|
+
}
|
|
344
|
+
return urls;
|
|
345
|
+
}
|
|
346
|
+
function readSitemap(raw) {
|
|
347
|
+
const value = raw.sitemap;
|
|
348
|
+
if (value === void 0) return [];
|
|
349
|
+
const list = typeof value === "string" ? [value] : value;
|
|
350
|
+
if (!Array.isArray(list) || !list.every((item) => typeof item === "string")) fail("\"sitemap\" must be a URL or an array of URLs.");
|
|
351
|
+
return list.map((item) => {
|
|
352
|
+
const url = withHttps(item);
|
|
353
|
+
if (!URL.canParse(url)) fail(`"sitemap" has an invalid URL: ${show(item)}.`);
|
|
354
|
+
return url;
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
function readArtifacts(raw) {
|
|
358
|
+
const value = raw.artifacts;
|
|
359
|
+
if (value === void 0) return [...CRAWL_DEFAULTS.artifacts];
|
|
360
|
+
const valid = `Valid artifacts: ${CRAWL_ARTIFACTS.join(", ")}.`;
|
|
361
|
+
if (!Array.isArray(value)) fail(`"artifacts" must be an array. ${valid}`);
|
|
362
|
+
for (const artifact of value) if (!CRAWL_ARTIFACTS.includes(artifact)) fail(`Invalid artifact ${show(artifact)}. ${valid}`);
|
|
363
|
+
return CRAWL_ARTIFACTS.filter((artifact) => value.includes(artifact));
|
|
364
|
+
}
|
|
365
|
+
function readDriver(raw) {
|
|
366
|
+
const value = raw.driver;
|
|
367
|
+
if (value === void 0) return CRAWL_DEFAULTS.driver;
|
|
368
|
+
if (!CRAWL_DRIVERS.includes(value)) fail(`"driver" must be "http" or "playwright". You gave ${show(value)}.`);
|
|
369
|
+
return value;
|
|
370
|
+
}
|
|
371
|
+
function readDepth(raw) {
|
|
372
|
+
const value = raw.depth;
|
|
373
|
+
if (value === void 0) return CRAWL_DEFAULTS.depth;
|
|
374
|
+
if (!Number.isInteger(value) || value < 0 || value > 10) fail(`"depth" must be an integer from 0 to 10. You gave ${show(value)}.`);
|
|
375
|
+
return value;
|
|
376
|
+
}
|
|
377
|
+
function readMaxPages(raw) {
|
|
378
|
+
const value = raw.maxPages;
|
|
379
|
+
if (value === void 0) return Number.POSITIVE_INFINITY;
|
|
380
|
+
if (!Number.isInteger(value) || value < 1) fail(`"maxPages" must be an integer of 1 or more. You gave ${show(value)}.`);
|
|
381
|
+
return value;
|
|
382
|
+
}
|
|
383
|
+
function readCrawlDelay(raw) {
|
|
384
|
+
const value = raw.crawlDelay;
|
|
385
|
+
if (value === void 0) return void 0;
|
|
386
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) fail(`"crawlDelay" must be a number of seconds, 0 or more. You gave ${show(value)}.`);
|
|
387
|
+
return value;
|
|
388
|
+
}
|
|
389
|
+
function readBoilerplateThreshold(raw) {
|
|
390
|
+
const value = raw.boilerplateThreshold;
|
|
391
|
+
if (value === void 0) return CRAWL_DEFAULTS.boilerplateThreshold;
|
|
392
|
+
if (typeof value !== "number" || !(value > 0 && value <= 1)) fail(`"boilerplateThreshold" must be a number greater than 0 and at most 1. You gave ${show(value)}.`);
|
|
393
|
+
return value;
|
|
394
|
+
}
|
|
395
|
+
function readExclude(raw) {
|
|
396
|
+
const exclude = readStrings(raw, "exclude") ?? [];
|
|
397
|
+
for (const pattern of exclude) {
|
|
398
|
+
const error = validateGlobPattern(pattern);
|
|
399
|
+
if (error) fail(`"exclude" has an invalid pattern. ${error}`);
|
|
400
|
+
}
|
|
401
|
+
return exclude;
|
|
402
|
+
}
|
|
403
|
+
function resolveCrawlOptions(options) {
|
|
404
|
+
if (typeof options !== "object" || options === null) fail("Crawl options must be an object.");
|
|
405
|
+
assertCrawlOptionKeys(options);
|
|
406
|
+
const raw = options;
|
|
407
|
+
const urls = readUrls(raw);
|
|
408
|
+
const globPatterns = readObject(raw, "globPatterns");
|
|
409
|
+
const silent = readBoolean(raw, "silent", false);
|
|
410
|
+
const logger = readObject(raw, "logger");
|
|
411
|
+
return {
|
|
412
|
+
urls,
|
|
413
|
+
patterns: globPatterns && globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern),
|
|
414
|
+
output: resolve(normalize(readString(raw, "output") ?? CRAWL_DEFAULTS.output)),
|
|
415
|
+
depth: readDepth(raw),
|
|
416
|
+
maxPages: readMaxPages(raw),
|
|
417
|
+
sitemap: readSitemap(raw),
|
|
418
|
+
artifacts: readArtifacts(raw),
|
|
419
|
+
siteName: readString(raw, "siteName"),
|
|
420
|
+
description: readString(raw, "description"),
|
|
421
|
+
origin: readString(raw, "origin"),
|
|
422
|
+
driver: readDriver(raw),
|
|
423
|
+
useChrome: readBoolean(raw, "useChrome", false),
|
|
424
|
+
exclude: readExclude(raw),
|
|
425
|
+
crawlDelay: readCrawlDelay(raw),
|
|
426
|
+
skipSitemap: readBoolean(raw, "skipSitemap", false),
|
|
427
|
+
allowSubdomains: readBoolean(raw, "allowSubdomains", false),
|
|
428
|
+
stripBoilerplate: readBoolean(raw, "stripBoilerplate", true),
|
|
429
|
+
boilerplateThreshold: readBoilerplateThreshold(raw),
|
|
430
|
+
verbose: readBoolean(raw, "verbose", false),
|
|
431
|
+
logger: resolveLogger({
|
|
432
|
+
silent,
|
|
433
|
+
logger
|
|
434
|
+
}),
|
|
435
|
+
hooks: readObject(raw, "hooks")
|
|
436
|
+
};
|
|
437
|
+
}
|
|
438
|
+
function mergeCrawlOptions(...layers) {
|
|
439
|
+
const merged = {};
|
|
440
|
+
const exclude = [];
|
|
441
|
+
for (const layer of layers) for (const key in layer) {
|
|
442
|
+
const value = layer[key];
|
|
443
|
+
if (value === void 0) continue;
|
|
444
|
+
if (key === "exclude" && Array.isArray(value)) exclude.push(...value);
|
|
445
|
+
else merged[key] = value;
|
|
446
|
+
}
|
|
447
|
+
if (exclude.length > 0) merged.exclude = exclude;
|
|
448
|
+
return merged;
|
|
449
|
+
}
|
|
239
450
|
const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
|
|
240
451
|
const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
|
|
241
452
|
const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
|
|
@@ -420,20 +631,17 @@ async function runConcurrent(items, concurrency, fn) {
|
|
|
420
631
|
await Promise.all(workers);
|
|
421
632
|
}
|
|
422
633
|
async function crawlAndGenerate(options, onProgress) {
|
|
423
|
-
|
|
424
|
-
|
|
634
|
+
return runCrawl(resolveCrawlOptions(options), onProgress);
|
|
635
|
+
}
|
|
636
|
+
async function runCrawl(options, onProgress) {
|
|
637
|
+
const { urls, patterns, output: outputDir, depth: maxDepth, maxPages, artifacts, origin, driver, useChrome, exclude, siteName: siteNameOption, description: descriptionOption, verbose, skipSitemap, sitemap: explicitSitemapUrls, allowSubdomains, stripBoilerplate, boilerplateThreshold, hooks: hooksConfig, logger } = options;
|
|
638
|
+
const writeLlmsTxt = artifacts.includes("llms.txt");
|
|
639
|
+
const writeLlmsFullTxt = artifacts.includes("llms-full.txt");
|
|
640
|
+
const writeMarkdown = artifacts.includes("markdown");
|
|
425
641
|
const hooks = createHooks();
|
|
426
642
|
if (hooksConfig) hooks.addHooks(hooksConfig);
|
|
427
|
-
if (onPage) hooks.hook("crawl:page", onPage);
|
|
428
643
|
const singlePageMode = maxDepth === 0;
|
|
429
|
-
|
|
430
|
-
let crawlDelay = userCrawlDelay;
|
|
431
|
-
let patterns;
|
|
432
|
-
try {
|
|
433
|
-
patterns = globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern);
|
|
434
|
-
} catch (error) {
|
|
435
|
-
throw new Error(`Invalid URL pattern: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
436
|
-
}
|
|
644
|
+
let crawlDelay = options.crawlDelay;
|
|
437
645
|
let startingUrls = patterns.map(getStartingUrl);
|
|
438
646
|
const hasGlobPatterns = patterns.some((p) => p.isGlob);
|
|
439
647
|
const progress = {
|
|
@@ -461,7 +669,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
461
669
|
if (startingUrls.length > 0 && !skipSitemap && !singlePageMode) {
|
|
462
670
|
const baseUrl = new URL(startingUrls[0]).origin;
|
|
463
671
|
const homePageUrl = baseUrl;
|
|
464
|
-
const hasExplicitSitemaps =
|
|
672
|
+
const hasExplicitSitemaps = explicitSitemapUrls.length > 0;
|
|
465
673
|
onProgress?.(progress);
|
|
466
674
|
let robotsContent = null;
|
|
467
675
|
try {
|
|
@@ -649,7 +857,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
649
857
|
title = pageData.title;
|
|
650
858
|
}
|
|
651
859
|
let filePath;
|
|
652
|
-
if (shouldProcessMarkdown &&
|
|
860
|
+
if (shouldProcessMarkdown && writeMarkdown) {
|
|
653
861
|
const urlPath = parsedUrl.pathname === "/" ? "/index" : parsedUrl.pathname;
|
|
654
862
|
const hostPrefix = allowSubdomains ? [parsedUrl.hostname.replace(URL_PATH_UNSAFE_CHARS_RE, "-")] : [];
|
|
655
863
|
const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
|
|
@@ -674,7 +882,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
674
882
|
progress.crawling.processed = results.length;
|
|
675
883
|
onProgress?.(progress);
|
|
676
884
|
}
|
|
677
|
-
if (
|
|
885
|
+
if (!singlePageMode && !sitemapProvidedUrls && depth < maxDepth) {
|
|
678
886
|
const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link) && isContentUrl(link));
|
|
679
887
|
for (const link of filteredLinks) {
|
|
680
888
|
if (processedUrls.has(link)) continue;
|
|
@@ -693,7 +901,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
693
901
|
}
|
|
694
902
|
}
|
|
695
903
|
};
|
|
696
|
-
const urlsToProcess = startingUrls.slice(0,
|
|
904
|
+
const urlsToProcess = startingUrls.slice(0, maxPages);
|
|
697
905
|
for (const url of urlsToProcess) processedUrls.add(url.replace(URL_TRAILING_SLASH_RE, "") || url);
|
|
698
906
|
let totalProcessed = 0;
|
|
699
907
|
progress.crawling.status = "processing";
|
|
@@ -739,7 +947,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
739
947
|
depth: request.userData?.depth || 0
|
|
740
948
|
});
|
|
741
949
|
},
|
|
742
|
-
maxRequestsPerCrawl,
|
|
950
|
+
maxRequestsPerCrawl: Number.isFinite(maxPages) ? maxPages : void 0,
|
|
743
951
|
respectRobotsTxtFile: false
|
|
744
952
|
};
|
|
745
953
|
if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
|
|
@@ -755,8 +963,8 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
755
963
|
try {
|
|
756
964
|
await crawler.run(allRequests);
|
|
757
965
|
totalProcessed += urlsToProcess.length;
|
|
758
|
-
while (pendingUrls.length > 0 && totalProcessed <
|
|
759
|
-
const batch = pendingUrls.splice(0,
|
|
966
|
+
while (pendingUrls.length > 0 && totalProcessed < maxPages) {
|
|
967
|
+
const batch = pendingUrls.splice(0, maxPages - totalProcessed);
|
|
760
968
|
progress.crawling.total += batch.length;
|
|
761
969
|
onProgress?.(progress);
|
|
762
970
|
const batchRequests = batch.map((item) => ({
|
|
@@ -836,8 +1044,8 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
836
1044
|
};
|
|
837
1045
|
await runConcurrent(urlsToProcess, DEFAULT_CONCURRENCY, (url) => fetchPage(url, 0));
|
|
838
1046
|
totalProcessed += urlsToProcess.length;
|
|
839
|
-
while (pendingUrls.length > 0 && totalProcessed <
|
|
840
|
-
const batch = pendingUrls.splice(0,
|
|
1047
|
+
while (pendingUrls.length > 0 && totalProcessed < maxPages) {
|
|
1048
|
+
const batch = pendingUrls.splice(0, maxPages - totalProcessed);
|
|
841
1049
|
progress.crawling.total += batch.length;
|
|
842
1050
|
onProgress?.(progress);
|
|
843
1051
|
await runConcurrent(batch, DEFAULT_CONCURRENCY, (item) => fetchPage(item.url, item.depth));
|
|
@@ -852,10 +1060,10 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
852
1060
|
onProgress?.(progress);
|
|
853
1061
|
const successfulResults = results.filter((r) => r.success);
|
|
854
1062
|
if (stripBoilerplate && !singlePageMode) {
|
|
855
|
-
const cleaned = stripBoilerplateFromCorpus(successfulResults.map((r) => r.content),
|
|
1063
|
+
const cleaned = stripBoilerplateFromCorpus(successfulResults.map((r) => r.content), { threshold: boilerplateThreshold });
|
|
856
1064
|
for (let i = 0; i < successfulResults.length; i++) successfulResults[i].content = cleaned[i];
|
|
857
1065
|
}
|
|
858
|
-
if (
|
|
1066
|
+
if (writeMarkdown) for (const result of successfulResults) {
|
|
859
1067
|
if (!result.filePath) continue;
|
|
860
1068
|
const contentCtx = {
|
|
861
1069
|
url: result.url,
|
|
@@ -879,9 +1087,9 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
879
1087
|
const resultUrl = new URL(withHttps(r.url));
|
|
880
1088
|
return resultUrl.href === originUrl || resultUrl.href === `${originUrl}/`;
|
|
881
1089
|
});
|
|
882
|
-
const siteName =
|
|
883
|
-
const description =
|
|
884
|
-
if (
|
|
1090
|
+
const siteName = siteNameOption || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
|
|
1091
|
+
const description = descriptionOption || homePageResult?.metadata?.description || successfulResults[0]?.metadata?.description;
|
|
1092
|
+
if (writeLlmsTxt || writeLlmsFullTxt) {
|
|
885
1093
|
progress.generation.current = "Generating llms.txt files";
|
|
886
1094
|
onProgress?.(progress);
|
|
887
1095
|
const contentResults = successfulResults.filter((result) => {
|
|
@@ -905,15 +1113,15 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
905
1113
|
siteName,
|
|
906
1114
|
description,
|
|
907
1115
|
origin: originUrl || firstUrl.origin,
|
|
908
|
-
generateFull:
|
|
1116
|
+
generateFull: writeLlmsFullTxt,
|
|
909
1117
|
outputDir
|
|
910
1118
|
});
|
|
911
|
-
if (
|
|
1119
|
+
if (writeLlmsTxt) {
|
|
912
1120
|
progress.generation.current = "Writing llms.txt";
|
|
913
1121
|
onProgress?.(progress);
|
|
914
1122
|
await writeFile(join(outputDir, "llms.txt"), llmsResult.llmsTxt, "utf-8");
|
|
915
1123
|
}
|
|
916
|
-
if (
|
|
1124
|
+
if (writeLlmsFullTxt && llmsResult.llmsFullTxt) {
|
|
917
1125
|
progress.generation.current = "Writing llms-full.txt";
|
|
918
1126
|
onProgress?.(progress);
|
|
919
1127
|
await writeFile(join(outputDir, "llms-full.txt"), llmsResult.llmsFullTxt, "utf-8");
|
|
@@ -924,4 +1132,4 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
924
1132
|
}
|
|
925
1133
|
return results;
|
|
926
1134
|
}
|
|
927
|
-
export {
|
|
1135
|
+
export { CRAWL_ARTIFACTS, CRAWL_DEFAULTS, assertCrawlOptionKeys, crawlAndGenerate, createClackLogger, mergeCrawlOptions, resolveCrawlOptions, resolveLogger, runCrawl, silentLogger, stripBoilerplateFromCorpus, validateCrawlUrl };
|