@mdream/crawl 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/crawl.mjs +38 -6
- package/dist/_chunks/playwright-utils.mjs +9 -7
- package/dist/cli.mjs +51 -33
- package/dist/index.d.mts +63 -1
- package/dist/index.mjs +2 -2
- package/package.json +5 -5
package/dist/_chunks/crawl.mjs
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { mkdirSync } from "node:fs";
|
|
2
2
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
3
|
-
import * as p from "@clack/prompts";
|
|
4
3
|
import { generateLlmsTxtArtifacts } from "@mdream/js/llms-txt";
|
|
5
4
|
import { createHooks } from "hookable";
|
|
6
5
|
import { htmlToMarkdown } from "mdream";
|
|
@@ -9,6 +8,7 @@ import { dirname, join, normalize, resolve } from "pathe";
|
|
|
9
8
|
import { withHttps } from "ufo";
|
|
10
9
|
import picomatch from "picomatch";
|
|
11
10
|
import { getDomain } from "tldts";
|
|
11
|
+
import * as p from "@clack/prompts";
|
|
12
12
|
function stripGlobTail(s) {
|
|
13
13
|
const idx = s.indexOf("*");
|
|
14
14
|
return idx === -1 ? s : s.slice(0, idx);
|
|
@@ -95,6 +95,37 @@ function validateGlobPattern(pattern) {
|
|
|
95
95
|
return `Invalid glob pattern: ${error instanceof Error ? error.message : error}`;
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
|
+
const noopSpinner = {
|
|
99
|
+
start() {},
|
|
100
|
+
message() {},
|
|
101
|
+
stop() {}
|
|
102
|
+
};
|
|
103
|
+
const silentLogger = {
|
|
104
|
+
intro() {},
|
|
105
|
+
note() {},
|
|
106
|
+
cancel() {},
|
|
107
|
+
info() {},
|
|
108
|
+
warn() {},
|
|
109
|
+
error() {},
|
|
110
|
+
success() {},
|
|
111
|
+
spinner: () => noopSpinner
|
|
112
|
+
};
|
|
113
|
+
function createClackLogger() {
|
|
114
|
+
return {
|
|
115
|
+
intro: (title) => p.intro(title),
|
|
116
|
+
note: (message, title) => p.note(message, title),
|
|
117
|
+
cancel: (message) => p.cancel(message),
|
|
118
|
+
info: (message) => p.log.info(message),
|
|
119
|
+
warn: (message) => p.log.warn(message),
|
|
120
|
+
error: (message) => p.log.error(message),
|
|
121
|
+
success: (message) => p.log.success(message),
|
|
122
|
+
spinner: () => p.spinner()
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
function resolveLogger(options) {
|
|
126
|
+
if (options.logger) return options.logger;
|
|
127
|
+
return options.silent ? silentLogger : createClackLogger();
|
|
128
|
+
}
|
|
98
129
|
const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
|
|
99
130
|
const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
|
|
100
131
|
const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
|
|
@@ -261,6 +292,7 @@ async function runConcurrent(items, concurrency, fn) {
|
|
|
261
292
|
}
|
|
262
293
|
async function crawlAndGenerate(options, onProgress) {
|
|
263
294
|
const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay: userCrawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, allowSubdomains = false, hooks: hooksConfig, onPage } = options;
|
|
295
|
+
const logger = resolveLogger(options);
|
|
264
296
|
const hooks = createHooks();
|
|
265
297
|
if (hooksConfig) hooks.addHooks(hooksConfig);
|
|
266
298
|
if (onPage) hooks.hook("crawl:page", onPage);
|
|
@@ -314,7 +346,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
314
346
|
const crawlDelayMatch = robotsContent.match(ROBOTS_CRAWL_DELAY_RE);
|
|
315
347
|
if (crawlDelayMatch) {
|
|
316
348
|
crawlDelay = Number.parseFloat(crawlDelayMatch[1]);
|
|
317
|
-
|
|
349
|
+
logger.info(`[ROBOTS] Crawl-delay: ${crawlDelay}s`);
|
|
318
350
|
}
|
|
319
351
|
}
|
|
320
352
|
if (robotsContent) {
|
|
@@ -418,9 +450,9 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
418
450
|
const failedSitemaps = sitemapAttempts.filter((a) => !a.success);
|
|
419
451
|
if (successfulSitemaps.length > 0 && progress.sitemap.processed > 0) {
|
|
420
452
|
sitemapProvidedUrls = true;
|
|
421
|
-
|
|
422
|
-
} else if (successfulSitemaps.length > 0)
|
|
423
|
-
else if (failedSitemaps.length > 0)
|
|
453
|
+
logger.info(`Sitemap: ${progress.sitemap.processed} URLs from ${successfulSitemaps[0].url}`);
|
|
454
|
+
} else if (successfulSitemaps.length > 0) logger.info(`Sitemap: found but no URLs matched filters`);
|
|
455
|
+
else if (failedSitemaps.length > 0) logger.info(`Sitemap: not found, discovering pages via crawler`);
|
|
424
456
|
if (!startingUrls.includes(homePageUrl)) startingUrls.unshift(homePageUrl);
|
|
425
457
|
progress.sitemap.status = "completed";
|
|
426
458
|
progress.crawling.total = startingUrls.length;
|
|
@@ -781,4 +813,4 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
781
813
|
}
|
|
782
814
|
return results;
|
|
783
815
|
}
|
|
784
|
-
export {
|
|
816
|
+
export { crawlAndGenerate, createClackLogger, parseUrlPattern, resolveLogger, silentLogger, validateGlobPattern };
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createClackLogger } from "./crawl.mjs";
|
|
1
2
|
import * as p from "@clack/prompts";
|
|
2
3
|
import { addDependency } from "nypm";
|
|
3
4
|
async function checkPlaywrightInstallation() {
|
|
@@ -8,13 +9,13 @@ async function checkPlaywrightInstallation() {
|
|
|
8
9
|
return false;
|
|
9
10
|
}
|
|
10
11
|
}
|
|
11
|
-
async function promptPlaywrightInstall() {
|
|
12
|
+
async function promptPlaywrightInstall(logger = createClackLogger()) {
|
|
12
13
|
const shouldInstall = await p.confirm({
|
|
13
14
|
message: "Playwright is required for the Playwright driver. Install it now?",
|
|
14
15
|
initialValue: true
|
|
15
16
|
});
|
|
16
17
|
if (p.isCancel(shouldInstall) || !shouldInstall) return false;
|
|
17
|
-
const s =
|
|
18
|
+
const s = logger.spinner();
|
|
18
19
|
s.start("Installing Playwright globally...");
|
|
19
20
|
try {
|
|
20
21
|
await addDependency("playwright", { global: true });
|
|
@@ -22,15 +23,16 @@ async function promptPlaywrightInstall() {
|
|
|
22
23
|
return true;
|
|
23
24
|
} catch (fallbackError) {
|
|
24
25
|
s.stop("Failed to install Playwright");
|
|
25
|
-
|
|
26
|
+
const reason = fallbackError instanceof Error ? fallbackError.message : String(fallbackError);
|
|
27
|
+
logger.error(`Installation failed: ${reason}`);
|
|
26
28
|
return false;
|
|
27
29
|
}
|
|
28
30
|
}
|
|
29
|
-
async function ensurePlaywrightInstalled() {
|
|
31
|
+
async function ensurePlaywrightInstalled(logger = createClackLogger()) {
|
|
30
32
|
if (await checkPlaywrightInstallation()) return true;
|
|
31
|
-
|
|
32
|
-
if (!await promptPlaywrightInstall()) {
|
|
33
|
-
|
|
33
|
+
logger.warn("Playwright driver selected but Playwright is not installed.");
|
|
34
|
+
if (!await promptPlaywrightInstall(logger)) {
|
|
35
|
+
logger.error("Cannot proceed with Playwright driver without Playwright installed.");
|
|
34
36
|
return false;
|
|
35
37
|
}
|
|
36
38
|
return true;
|
package/dist/cli.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { crawlAndGenerate, parseUrlPattern, resolveLogger, validateGlobPattern } from "./_chunks/crawl.mjs";
|
|
2
2
|
import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
|
-
import * as p from "@clack/prompts";
|
|
4
3
|
import { dirname, join, relative, resolve } from "pathe";
|
|
5
4
|
import { withHttps } from "ufo";
|
|
5
|
+
import * as p from "@clack/prompts";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import { loadConfig } from "c12";
|
|
8
8
|
async function loadMdreamConfig(cwd) {
|
|
@@ -12,6 +12,15 @@ async function loadMdreamConfig(cwd) {
|
|
|
12
12
|
});
|
|
13
13
|
return config || {};
|
|
14
14
|
}
|
|
15
|
+
const QUIET_FLAGS = new Set([
|
|
16
|
+
"--silent",
|
|
17
|
+
"--quiet",
|
|
18
|
+
"-q"
|
|
19
|
+
]);
|
|
20
|
+
function isSilentArgv(args) {
|
|
21
|
+
return args.some((arg) => QUIET_FLAGS.has(arg));
|
|
22
|
+
}
|
|
23
|
+
let runLogger = resolveLogger({ silent: isSilentArgv(process.argv.slice(2)) });
|
|
15
24
|
const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
|
|
16
25
|
const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
|
|
17
26
|
function checkOutputDirectoryPermissions(outputDir) {
|
|
@@ -175,7 +184,7 @@ async function interactiveCrawl() {
|
|
|
175
184
|
skipSitemap: advancedOptions.skipSitemap
|
|
176
185
|
};
|
|
177
186
|
}
|
|
178
|
-
async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds, latency) {
|
|
187
|
+
async function showCrawlResults(logger, successful, failed, outputDir, generatedFiles, durationSeconds, latency) {
|
|
179
188
|
let line = `${successful} pages in ${`${durationSeconds.toFixed(1)}s`}`;
|
|
180
189
|
if (failed > 0) line += ` (${failed} failed)`;
|
|
181
190
|
if (latency && latency.count > 0) {
|
|
@@ -184,11 +193,13 @@ async function showCrawlResults(successful, failed, outputDir, generatedFiles, d
|
|
|
184
193
|
const max = Math.round(latency.max);
|
|
185
194
|
line += ` \u00B7 HTTP Latency: avg ${avg}ms, min ${min}ms, max ${max}ms`;
|
|
186
195
|
}
|
|
187
|
-
|
|
188
|
-
|
|
196
|
+
logger.success(line);
|
|
197
|
+
logger.info(`${generatedFiles.join(", ")} \u2192 ${relative(process.cwd(), outputDir) || "."}`);
|
|
189
198
|
}
|
|
190
199
|
function parseCliArgs() {
|
|
191
200
|
const args = process.argv.slice(2);
|
|
201
|
+
const silent = isSilentArgv(args);
|
|
202
|
+
const logger = resolveLogger({ silent });
|
|
192
203
|
if (args.includes("--help") || args.includes("-h")) {
|
|
193
204
|
console.log(`
|
|
194
205
|
@mdream/crawl v${version}
|
|
@@ -215,6 +226,7 @@ Options:
|
|
|
215
226
|
--skip-sitemap Skip sitemap.xml and robots.txt discovery
|
|
216
227
|
--allow-subdomains Crawl across subdomains of the same root domain
|
|
217
228
|
-v, --verbose Enable verbose logging
|
|
229
|
+
-q, --quiet, --silent Suppress all logs (clean stdout for JSON-RPC/MCP)
|
|
218
230
|
-h, --help Show this help message
|
|
219
231
|
--version Show version number
|
|
220
232
|
|
|
@@ -250,52 +262,52 @@ Examples:
|
|
|
250
262
|
const urlFromArgs = args.find((arg) => !arg.startsWith("-") && !args[args.indexOf(arg) - 1]?.startsWith("-"));
|
|
251
263
|
const url = urlFromFlag || urlFromArgs;
|
|
252
264
|
if (!url) {
|
|
253
|
-
|
|
254
|
-
|
|
265
|
+
logger.error("Error: URL is required when using CLI arguments");
|
|
266
|
+
logger.info("Use --help for usage information or run without arguments for interactive mode");
|
|
255
267
|
process.exit(1);
|
|
256
268
|
}
|
|
257
269
|
const globError = validateGlobPattern(url);
|
|
258
270
|
if (globError) {
|
|
259
|
-
|
|
271
|
+
logger.error(`Error: ${globError}`);
|
|
260
272
|
process.exit(1);
|
|
261
273
|
}
|
|
262
274
|
let parsed;
|
|
263
275
|
try {
|
|
264
276
|
parsed = parseUrlPattern(url);
|
|
265
277
|
} catch (error) {
|
|
266
|
-
|
|
278
|
+
logger.error(`Error: ${error instanceof Error ? error.message : "Invalid URL pattern"}`);
|
|
267
279
|
process.exit(1);
|
|
268
280
|
}
|
|
269
281
|
if (!parsed.isGlob) try {
|
|
270
282
|
new URL(withHttps(url));
|
|
271
283
|
} catch {
|
|
272
|
-
|
|
284
|
+
logger.error(`Error: Invalid URL: ${withHttps(url)}`);
|
|
273
285
|
process.exit(1);
|
|
274
286
|
}
|
|
275
287
|
const excludePatterns = getArgValues("--exclude");
|
|
276
288
|
for (const pattern of excludePatterns) {
|
|
277
289
|
const excludeError = validateGlobPattern(pattern);
|
|
278
290
|
if (excludeError) {
|
|
279
|
-
|
|
291
|
+
logger.error(`Error in exclude pattern: ${excludeError}`);
|
|
280
292
|
process.exit(1);
|
|
281
293
|
}
|
|
282
294
|
}
|
|
283
295
|
const depthStr = args.includes("--single-page") ? "0" : getArgValue("--depth") || getArgValue("-d") || "3";
|
|
284
296
|
const depth = Number(depthStr);
|
|
285
297
|
if (!Number.isInteger(depth) || depth < 0 || depth > 10) {
|
|
286
|
-
|
|
298
|
+
logger.error("Error: Depth must be an integer between 0 and 10");
|
|
287
299
|
process.exit(1);
|
|
288
300
|
}
|
|
289
301
|
const driver = getArgValue("--driver");
|
|
290
302
|
if (driver && driver !== "http" && driver !== "playwright") {
|
|
291
|
-
|
|
303
|
+
logger.error("Error: Driver must be either \"http\" or \"playwright\"");
|
|
292
304
|
process.exit(1);
|
|
293
305
|
}
|
|
294
306
|
const maxPagesStr = getArgValue("--max-pages");
|
|
295
307
|
if (maxPagesStr) {
|
|
296
308
|
const maxPages = Number.parseInt(maxPagesStr);
|
|
297
309
|
if (Number.isNaN(maxPages) || maxPages < 1) {
|
|
298
|
-
|
|
310
|
+
logger.error("Error: Max pages must be a positive number");
|
|
299
311
|
process.exit(1);
|
|
300
312
|
}
|
|
301
313
|
}
|
|
@@ -303,7 +315,7 @@ Examples:
|
|
|
303
315
|
if (crawlDelayStr) {
|
|
304
316
|
const crawlDelay = Number.parseInt(crawlDelayStr);
|
|
305
317
|
if (Number.isNaN(crawlDelay) || crawlDelay < 0) {
|
|
306
|
-
|
|
318
|
+
logger.error("Error: Crawl delay must be a non-negative number");
|
|
307
319
|
process.exit(1);
|
|
308
320
|
}
|
|
309
321
|
}
|
|
@@ -319,7 +331,7 @@ Examples:
|
|
|
319
331
|
"markdown"
|
|
320
332
|
];
|
|
321
333
|
for (const artifact of artifacts) if (!validArtifacts.includes(artifact)) {
|
|
322
|
-
|
|
334
|
+
logger.error(`Error: Invalid artifact '${artifact}'. Valid options: ${validArtifacts.join(", ")}`);
|
|
323
335
|
process.exit(1);
|
|
324
336
|
}
|
|
325
337
|
const originOverride = getArgValue("--origin");
|
|
@@ -338,7 +350,7 @@ Examples:
|
|
|
338
350
|
const verbose = args.includes("--verbose") || args.includes("-v");
|
|
339
351
|
const skipSitemap = args.includes("--skip-sitemap");
|
|
340
352
|
const allowSubdomains = args.includes("--allow-subdomains");
|
|
341
|
-
if (skipSitemap && parsed.isGlob)
|
|
353
|
+
if (skipSitemap && parsed.isGlob) logger.warn("Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.");
|
|
342
354
|
return {
|
|
343
355
|
urls: [url],
|
|
344
356
|
outputDir: resolve(getArgValue("--output") || getArgValue("-o") || "output"),
|
|
@@ -357,12 +369,16 @@ Examples:
|
|
|
357
369
|
exclude: excludePatterns.length > 0 ? excludePatterns : void 0,
|
|
358
370
|
verbose,
|
|
359
371
|
skipSitemap,
|
|
360
|
-
allowSubdomains
|
|
372
|
+
allowSubdomains,
|
|
373
|
+
silent
|
|
361
374
|
};
|
|
362
375
|
}
|
|
363
376
|
async function main() {
|
|
364
377
|
const cliOptions = parseCliArgs();
|
|
365
378
|
const fileConfig = await loadMdreamConfig();
|
|
379
|
+
const silent = (cliOptions?.silent ?? false) || (fileConfig.silent ?? false);
|
|
380
|
+
const logger = resolveLogger({ silent });
|
|
381
|
+
runLogger = logger;
|
|
366
382
|
let options;
|
|
367
383
|
if (cliOptions) {
|
|
368
384
|
const configExclude = fileConfig.exclude || [];
|
|
@@ -375,10 +391,11 @@ async function main() {
|
|
|
375
391
|
skipSitemap: cliOptions.skipSitemap || fileConfig.skipSitemap || false,
|
|
376
392
|
allowSubdomains: cliOptions.allowSubdomains || fileConfig.allowSubdomains || false,
|
|
377
393
|
verbose: cliOptions.verbose || fileConfig.verbose || false,
|
|
394
|
+
silent,
|
|
378
395
|
exclude: configExclude.length > 0 || cliExclude.length > 0 ? [...configExclude, ...cliExclude] : void 0,
|
|
379
396
|
hooks: fileConfig.hooks
|
|
380
397
|
};
|
|
381
|
-
|
|
398
|
+
logger.intro(`☁️ mdream v${version}`);
|
|
382
399
|
const formats = [];
|
|
383
400
|
if (options.generateLlmsTxt) formats.push("llms.txt");
|
|
384
401
|
if (options.generateLlmsFullTxt) formats.push("llms-full.txt");
|
|
@@ -393,35 +410,35 @@ async function main() {
|
|
|
393
410
|
options.allowSubdomains && `Allow subdomains: Yes`,
|
|
394
411
|
options.verbose && `Verbose: Enabled`
|
|
395
412
|
].filter(Boolean);
|
|
396
|
-
|
|
413
|
+
logger.note(summary.join("\n"), "Configuration");
|
|
397
414
|
} else options = await interactiveCrawl();
|
|
398
415
|
if (!options) process.exit(0);
|
|
399
416
|
const permCheck = checkOutputDirectoryPermissions(options.outputDir);
|
|
400
417
|
if (!permCheck.success) {
|
|
401
|
-
|
|
402
|
-
if (permCheck.error?.includes("Permission denied"))
|
|
418
|
+
logger.error(permCheck.error);
|
|
419
|
+
if (permCheck.error?.includes("Permission denied")) logger.info("Tip: Try running with elevated privileges (e.g., sudo) or change the output directory permissions.");
|
|
403
420
|
process.exit(1);
|
|
404
421
|
}
|
|
405
422
|
if (options.driver === "playwright") {
|
|
406
423
|
try {
|
|
407
424
|
await import("crawlee");
|
|
408
425
|
} catch {
|
|
409
|
-
|
|
426
|
+
logger.error("The Playwright driver requires crawlee. Install it with: npm install crawlee");
|
|
410
427
|
process.exit(1);
|
|
411
428
|
}
|
|
412
429
|
const { ensurePlaywrightInstalled, isUseChromeSupported } = await import("./_chunks/playwright-utils.mjs");
|
|
413
430
|
if (await isUseChromeSupported()) {
|
|
414
431
|
options.useChrome = true;
|
|
415
|
-
|
|
432
|
+
logger.info("System Chrome detected and enabled.");
|
|
416
433
|
} else {
|
|
417
|
-
if (!await ensurePlaywrightInstalled()) {
|
|
418
|
-
|
|
434
|
+
if (!await ensurePlaywrightInstalled(logger)) {
|
|
435
|
+
logger.error("Cannot proceed without Playwright. Please install it manually or use the HTTP driver instead.");
|
|
419
436
|
process.exit(1);
|
|
420
437
|
}
|
|
421
|
-
|
|
438
|
+
logger.info("Using global playwright instance.");
|
|
422
439
|
}
|
|
423
440
|
}
|
|
424
|
-
const s =
|
|
441
|
+
const s = logger.spinner();
|
|
425
442
|
s.start("Discovering sitemaps");
|
|
426
443
|
const startTime = Date.now();
|
|
427
444
|
let crawlStartTime = 0;
|
|
@@ -449,9 +466,9 @@ async function main() {
|
|
|
449
466
|
const failed = results.filter((r) => !r.success).length;
|
|
450
467
|
const failedResults = results.filter((r) => !r.success);
|
|
451
468
|
if (failed > 0 && cliOptions) {
|
|
452
|
-
|
|
469
|
+
logger.error("Failed URLs:");
|
|
453
470
|
failedResults.forEach((result) => {
|
|
454
|
-
|
|
471
|
+
logger.error(` ${result.url}: ${result.error || "Unknown error"}`);
|
|
455
472
|
});
|
|
456
473
|
} else if (failed > 0) {
|
|
457
474
|
console.log("\nFailed URLs:");
|
|
@@ -465,13 +482,14 @@ async function main() {
|
|
|
465
482
|
if (options.generateLlmsFullTxt) generatedFiles.push("llms-full.txt");
|
|
466
483
|
if (options.generateIndividualMd) generatedFiles.push(`${successful} MD files`);
|
|
467
484
|
}
|
|
468
|
-
await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationSeconds, lastProgress?.crawling.latency);
|
|
485
|
+
await showCrawlResults(logger, successful, failed, options.outputDir, generatedFiles, durationSeconds, lastProgress?.crawling.latency);
|
|
469
486
|
process.exit(0);
|
|
470
487
|
}
|
|
471
488
|
main().catch((error) => {
|
|
489
|
+
const logger = runLogger;
|
|
472
490
|
const msg = error instanceof Error ? error.message : String(error);
|
|
473
|
-
if (msg.includes("wmic") || msg.includes("ENOENT") && process.platform === "win32")
|
|
474
|
-
else
|
|
491
|
+
if (msg.includes("wmic") || msg.includes("ENOENT") && process.platform === "win32") logger.error("Crawlee failed because wmic.exe is not available on this system. Windows 11 removed wmic.exe, which older crawlee versions depend on for memory monitoring.\nFix: upgrade crawlee to >=3.16.0 or switch to the HTTP driver (--driver http).");
|
|
492
|
+
else logger.error(`Unexpected error: ${msg}`);
|
|
475
493
|
process.exit(1);
|
|
476
494
|
});
|
|
477
495
|
export {};
|
package/dist/index.d.mts
CHANGED
|
@@ -1,3 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal spinner surface used by the crawler. Mirrors the subset of
|
|
3
|
+
* `@clack/prompts` spinner methods the package relies on.
|
|
4
|
+
*/
|
|
5
|
+
interface CrawlSpinner {
|
|
6
|
+
start: (message?: string) => void;
|
|
7
|
+
message: (message?: string) => void;
|
|
8
|
+
stop: (message?: string, code?: number) => void;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Diagnostic/progress sink for the crawler. All human-facing, non-data output
|
|
12
|
+
* (the messages that would otherwise pollute stdout) flows through this single
|
|
13
|
+
* seam so it can be silenced or redirected. It deliberately excludes the
|
|
14
|
+
* interactive prompts (`select`, `text`, `confirm`, ...) which are input
|
|
15
|
+
* control flow, not log output, and only run in the interactive CLI.
|
|
16
|
+
*
|
|
17
|
+
* Provide a custom implementation via `CrawlOptions.logger` to route messages
|
|
18
|
+
* anywhere (e.g. stderr for an MCP server), or set `CrawlOptions.silent` to
|
|
19
|
+
* drop them entirely. See issue #100.
|
|
20
|
+
*/
|
|
21
|
+
interface CrawlLogger {
|
|
22
|
+
intro: (title?: string) => void;
|
|
23
|
+
note: (message: string, title?: string) => void;
|
|
24
|
+
cancel: (message?: string) => void;
|
|
25
|
+
info: (message: string) => void;
|
|
26
|
+
warn: (message: string) => void;
|
|
27
|
+
error: (message: string) => void;
|
|
28
|
+
success: (message: string) => void;
|
|
29
|
+
spinner: () => CrawlSpinner;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* A logger that drops every message. Used when `silent` is set so a library
|
|
33
|
+
* consumer (e.g. an MCP server) gets a clean stdout.
|
|
34
|
+
*/
|
|
35
|
+
declare const silentLogger: CrawlLogger;
|
|
36
|
+
/**
|
|
37
|
+
* The default logger, backed by `@clack/prompts` (writes to stdout). Preserves
|
|
38
|
+
* the existing CLI output exactly.
|
|
39
|
+
*/
|
|
40
|
+
declare function createClackLogger(): CrawlLogger;
|
|
41
|
+
/**
|
|
42
|
+
* Resolve the effective logger from crawl options. An explicit `logger` wins;
|
|
43
|
+
* otherwise `silent` selects the no-op sink, and the default is the clack
|
|
44
|
+
* logger. Pure: same inputs, same logger.
|
|
45
|
+
*/
|
|
46
|
+
declare function resolveLogger(options: {
|
|
47
|
+
silent?: boolean;
|
|
48
|
+
logger?: CrawlLogger;
|
|
49
|
+
}): CrawlLogger;
|
|
1
50
|
interface PageData {
|
|
2
51
|
url: string;
|
|
3
52
|
html: string;
|
|
@@ -47,6 +96,17 @@ interface CrawlOptions {
|
|
|
47
96
|
verbose?: boolean;
|
|
48
97
|
skipSitemap?: boolean;
|
|
49
98
|
allowSubdomains?: boolean;
|
|
99
|
+
/**
|
|
100
|
+
* Suppress all diagnostic/progress logging. Use when stdout must stay clean,
|
|
101
|
+
* e.g. an MCP server that only emits JSON-RPC (issue #100). Ignored when an
|
|
102
|
+
* explicit `logger` is provided.
|
|
103
|
+
*/
|
|
104
|
+
silent?: boolean;
|
|
105
|
+
/**
|
|
106
|
+
* Custom sink for diagnostic/progress messages. Route logs anywhere (e.g.
|
|
107
|
+
* stderr) instead of the default `@clack/prompts` stdout output.
|
|
108
|
+
*/
|
|
109
|
+
logger?: CrawlLogger;
|
|
50
110
|
hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][] }>;
|
|
51
111
|
onPage?: (page: PageData) => Promise<void> | void;
|
|
52
112
|
}
|
|
@@ -59,6 +119,8 @@ interface MdreamCrawlConfig {
|
|
|
59
119
|
skipSitemap?: boolean;
|
|
60
120
|
allowSubdomains?: boolean;
|
|
61
121
|
verbose?: boolean;
|
|
122
|
+
/** Suppress all diagnostic/progress logging (issue #100). */
|
|
123
|
+
silent?: boolean;
|
|
62
124
|
artifacts?: ('llms.txt' | 'llms-full.txt' | 'markdown')[];
|
|
63
125
|
hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][] }>;
|
|
64
126
|
}
|
|
@@ -111,4 +173,4 @@ interface CrawlProgress {
|
|
|
111
173
|
};
|
|
112
174
|
}
|
|
113
175
|
declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
|
|
114
|
-
export { type CrawlHooks, type CrawlOptions, type CrawlResult, type MdreamCrawlConfig, type PageData, crawlAndGenerate, defineConfig };
|
|
176
|
+
export { type CrawlHooks, type CrawlLogger, type CrawlOptions, type CrawlResult, type CrawlSpinner, type MdreamCrawlConfig, type PageData, crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger };
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { crawlAndGenerate, createClackLogger, resolveLogger, silentLogger } from "./_chunks/crawl.mjs";
|
|
2
2
|
function defineConfig(config) {
|
|
3
3
|
return config;
|
|
4
4
|
}
|
|
5
|
-
export { crawlAndGenerate, defineConfig };
|
|
5
|
+
export { crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/crawl",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.3.0",
|
|
5
5
|
"description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -54,17 +54,17 @@
|
|
|
54
54
|
}
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
|
-
"@clack/prompts": "^1.
|
|
57
|
+
"@clack/prompts": "^1.5.1",
|
|
58
58
|
"c12": "^3.3.4",
|
|
59
59
|
"hookable": "^6.1.1",
|
|
60
60
|
"nypm": "^0.6.6",
|
|
61
61
|
"ofetch": "^1.5.1",
|
|
62
62
|
"pathe": "^2.0.3",
|
|
63
63
|
"picomatch": "^4.0.4",
|
|
64
|
-
"tldts": "^7.
|
|
64
|
+
"tldts": "^7.4.2",
|
|
65
65
|
"ufo": "^1.6.4",
|
|
66
|
-
"@mdream/js": "1.
|
|
67
|
-
"mdream": "1.
|
|
66
|
+
"@mdream/js": "1.3.0",
|
|
67
|
+
"mdream": "1.3.0"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
70
|
"@types/picomatch": "^4.0.3"
|