@mdream/crawl 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,5 @@
1
1
  import { mkdirSync } from "node:fs";
2
2
  import { mkdir, writeFile } from "node:fs/promises";
3
- import * as p from "@clack/prompts";
4
3
  import { generateLlmsTxtArtifacts } from "@mdream/js/llms-txt";
5
4
  import { createHooks } from "hookable";
6
5
  import { htmlToMarkdown } from "mdream";
@@ -9,6 +8,7 @@ import { dirname, join, normalize, resolve } from "pathe";
9
8
  import { withHttps } from "ufo";
10
9
  import picomatch from "picomatch";
11
10
  import { getDomain } from "tldts";
11
+ import * as p from "@clack/prompts";
12
12
  function stripGlobTail(s) {
13
13
  const idx = s.indexOf("*");
14
14
  return idx === -1 ? s : s.slice(0, idx);
@@ -95,6 +95,37 @@ function validateGlobPattern(pattern) {
95
95
  return `Invalid glob pattern: ${error instanceof Error ? error.message : error}`;
96
96
  }
97
97
  }
98
+ const noopSpinner = {
99
+ start() {},
100
+ message() {},
101
+ stop() {}
102
+ };
103
+ const silentLogger = {
104
+ intro() {},
105
+ note() {},
106
+ cancel() {},
107
+ info() {},
108
+ warn() {},
109
+ error() {},
110
+ success() {},
111
+ spinner: () => noopSpinner
112
+ };
113
+ function createClackLogger() {
114
+ return {
115
+ intro: (title) => p.intro(title),
116
+ note: (message, title) => p.note(message, title),
117
+ cancel: (message) => p.cancel(message),
118
+ info: (message) => p.log.info(message),
119
+ warn: (message) => p.log.warn(message),
120
+ error: (message) => p.log.error(message),
121
+ success: (message) => p.log.success(message),
122
+ spinner: () => p.spinner()
123
+ };
124
+ }
125
+ function resolveLogger(options) {
126
+ if (options.logger) return options.logger;
127
+ return options.silent ? silentLogger : createClackLogger();
128
+ }
98
129
  const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
99
130
  const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
100
131
  const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
@@ -261,6 +292,7 @@ async function runConcurrent(items, concurrency, fn) {
261
292
  }
262
293
  async function crawlAndGenerate(options, onProgress) {
263
294
  const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay: userCrawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, allowSubdomains = false, hooks: hooksConfig, onPage } = options;
295
+ const logger = resolveLogger(options);
264
296
  const hooks = createHooks();
265
297
  if (hooksConfig) hooks.addHooks(hooksConfig);
266
298
  if (onPage) hooks.hook("crawl:page", onPage);
@@ -314,7 +346,7 @@ async function crawlAndGenerate(options, onProgress) {
314
346
  const crawlDelayMatch = robotsContent.match(ROBOTS_CRAWL_DELAY_RE);
315
347
  if (crawlDelayMatch) {
316
348
  crawlDelay = Number.parseFloat(crawlDelayMatch[1]);
317
- p.log.info(`[ROBOTS] Crawl-delay: ${crawlDelay}s`);
349
+ logger.info(`[ROBOTS] Crawl-delay: ${crawlDelay}s`);
318
350
  }
319
351
  }
320
352
  if (robotsContent) {
@@ -418,9 +450,9 @@ async function crawlAndGenerate(options, onProgress) {
418
450
  const failedSitemaps = sitemapAttempts.filter((a) => !a.success);
419
451
  if (successfulSitemaps.length > 0 && progress.sitemap.processed > 0) {
420
452
  sitemapProvidedUrls = true;
421
- p.log.info(`Sitemap: ${progress.sitemap.processed} URLs from ${successfulSitemaps[0].url}`);
422
- } else if (successfulSitemaps.length > 0) p.log.info(`Sitemap: found but no URLs matched filters`);
423
- else if (failedSitemaps.length > 0) p.log.info(`Sitemap: not found, discovering pages via crawler`);
453
+ logger.info(`Sitemap: ${progress.sitemap.processed} URLs from ${successfulSitemaps[0].url}`);
454
+ } else if (successfulSitemaps.length > 0) logger.info(`Sitemap: found but no URLs matched filters`);
455
+ else if (failedSitemaps.length > 0) logger.info(`Sitemap: not found, discovering pages via crawler`);
424
456
  if (!startingUrls.includes(homePageUrl)) startingUrls.unshift(homePageUrl);
425
457
  progress.sitemap.status = "completed";
426
458
  progress.crawling.total = startingUrls.length;
@@ -781,4 +813,4 @@ async function crawlAndGenerate(options, onProgress) {
781
813
  }
782
814
  return results;
783
815
  }
784
- export { parseUrlPattern as n, validateGlobPattern as r, crawlAndGenerate as t };
816
+ export { crawlAndGenerate, createClackLogger, parseUrlPattern, resolveLogger, silentLogger, validateGlobPattern };
@@ -1,3 +1,4 @@
1
+ import { createClackLogger } from "./crawl.mjs";
1
2
  import * as p from "@clack/prompts";
2
3
  import { addDependency } from "nypm";
3
4
  async function checkPlaywrightInstallation() {
@@ -8,13 +9,13 @@ async function checkPlaywrightInstallation() {
8
9
  return false;
9
10
  }
10
11
  }
11
- async function promptPlaywrightInstall() {
12
+ async function promptPlaywrightInstall(logger = createClackLogger()) {
12
13
  const shouldInstall = await p.confirm({
13
14
  message: "Playwright is required for the Playwright driver. Install it now?",
14
15
  initialValue: true
15
16
  });
16
17
  if (p.isCancel(shouldInstall) || !shouldInstall) return false;
17
- const s = p.spinner();
18
+ const s = logger.spinner();
18
19
  s.start("Installing Playwright globally...");
19
20
  try {
20
21
  await addDependency("playwright", { global: true });
@@ -22,15 +23,16 @@ async function promptPlaywrightInstall() {
22
23
  return true;
23
24
  } catch (fallbackError) {
24
25
  s.stop("Failed to install Playwright");
25
- p.log.error(`Installation failed: ${fallbackError}`);
26
+ const reason = fallbackError instanceof Error ? fallbackError.message : String(fallbackError);
27
+ logger.error(`Installation failed: ${reason}`);
26
28
  return false;
27
29
  }
28
30
  }
29
- async function ensurePlaywrightInstalled() {
31
+ async function ensurePlaywrightInstalled(logger = createClackLogger()) {
30
32
  if (await checkPlaywrightInstallation()) return true;
31
- p.log.warn("Playwright driver selected but Playwright is not installed.");
32
- if (!await promptPlaywrightInstall()) {
33
- p.log.error("Cannot proceed with Playwright driver without Playwright installed.");
33
+ logger.warn("Playwright driver selected but Playwright is not installed.");
34
+ if (!await promptPlaywrightInstall(logger)) {
35
+ logger.error("Cannot proceed with Playwright driver without Playwright installed.");
34
36
  return false;
35
37
  }
36
38
  return true;
package/dist/cli.mjs CHANGED
@@ -1,8 +1,8 @@
1
- import { n as parseUrlPattern, r as validateGlobPattern, t as crawlAndGenerate } from "./_chunks/crawl.mjs";
1
+ import { crawlAndGenerate, parseUrlPattern, resolveLogger, validateGlobPattern } from "./_chunks/crawl.mjs";
2
2
  import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
3
- import * as p from "@clack/prompts";
4
3
  import { dirname, join, relative, resolve } from "pathe";
5
4
  import { withHttps } from "ufo";
5
+ import * as p from "@clack/prompts";
6
6
  import { fileURLToPath } from "node:url";
7
7
  import { loadConfig } from "c12";
8
8
  async function loadMdreamConfig(cwd) {
@@ -12,6 +12,15 @@ async function loadMdreamConfig(cwd) {
12
12
  });
13
13
  return config || {};
14
14
  }
15
+ const QUIET_FLAGS = new Set([
16
+ "--silent",
17
+ "--quiet",
18
+ "-q"
19
+ ]);
20
+ function isSilentArgv(args) {
21
+ return args.some((arg) => QUIET_FLAGS.has(arg));
22
+ }
23
+ let runLogger = resolveLogger({ silent: isSilentArgv(process.argv.slice(2)) });
15
24
  const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
16
25
  const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
17
26
  function checkOutputDirectoryPermissions(outputDir) {
@@ -175,7 +184,7 @@ async function interactiveCrawl() {
175
184
  skipSitemap: advancedOptions.skipSitemap
176
185
  };
177
186
  }
178
- async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds, latency) {
187
+ async function showCrawlResults(logger, successful, failed, outputDir, generatedFiles, durationSeconds, latency) {
179
188
  let line = `${successful} pages in ${`${durationSeconds.toFixed(1)}s`}`;
180
189
  if (failed > 0) line += ` (${failed} failed)`;
181
190
  if (latency && latency.count > 0) {
@@ -184,11 +193,13 @@ async function showCrawlResults(successful, failed, outputDir, generatedFiles, d
184
193
  const max = Math.round(latency.max);
185
194
  line += ` \u00B7 HTTP Latency: avg ${avg}ms, min ${min}ms, max ${max}ms`;
186
195
  }
187
- p.log.success(line);
188
- p.log.info(`${generatedFiles.join(", ")} \u2192 ${relative(process.cwd(), outputDir) || "."}`);
196
+ logger.success(line);
197
+ logger.info(`${generatedFiles.join(", ")} \u2192 ${relative(process.cwd(), outputDir) || "."}`);
189
198
  }
190
199
  function parseCliArgs() {
191
200
  const args = process.argv.slice(2);
201
+ const silent = isSilentArgv(args);
202
+ const logger = resolveLogger({ silent });
192
203
  if (args.includes("--help") || args.includes("-h")) {
193
204
  console.log(`
194
205
  @mdream/crawl v${version}
@@ -215,6 +226,7 @@ Options:
215
226
  --skip-sitemap Skip sitemap.xml and robots.txt discovery
216
227
  --allow-subdomains Crawl across subdomains of the same root domain
217
228
  -v, --verbose Enable verbose logging
229
+ -q, --quiet, --silent Suppress all logs (clean stdout for JSON-RPC/MCP)
218
230
  -h, --help Show this help message
219
231
  --version Show version number
220
232
 
@@ -250,52 +262,52 @@ Examples:
250
262
  const urlFromArgs = args.find((arg) => !arg.startsWith("-") && !args[args.indexOf(arg) - 1]?.startsWith("-"));
251
263
  const url = urlFromFlag || urlFromArgs;
252
264
  if (!url) {
253
- p.log.error("Error: URL is required when using CLI arguments");
254
- p.log.info("Use --help for usage information or run without arguments for interactive mode");
265
+ logger.error("Error: URL is required when using CLI arguments");
266
+ logger.info("Use --help for usage information or run without arguments for interactive mode");
255
267
  process.exit(1);
256
268
  }
257
269
  const globError = validateGlobPattern(url);
258
270
  if (globError) {
259
- p.log.error(`Error: ${globError}`);
271
+ logger.error(`Error: ${globError}`);
260
272
  process.exit(1);
261
273
  }
262
274
  let parsed;
263
275
  try {
264
276
  parsed = parseUrlPattern(url);
265
277
  } catch (error) {
266
- p.log.error(`Error: ${error instanceof Error ? error.message : "Invalid URL pattern"}`);
278
+ logger.error(`Error: ${error instanceof Error ? error.message : "Invalid URL pattern"}`);
267
279
  process.exit(1);
268
280
  }
269
281
  if (!parsed.isGlob) try {
270
282
  new URL(withHttps(url));
271
283
  } catch {
272
- p.log.error(`Error: Invalid URL: ${withHttps(url)}`);
284
+ logger.error(`Error: Invalid URL: ${withHttps(url)}`);
273
285
  process.exit(1);
274
286
  }
275
287
  const excludePatterns = getArgValues("--exclude");
276
288
  for (const pattern of excludePatterns) {
277
289
  const excludeError = validateGlobPattern(pattern);
278
290
  if (excludeError) {
279
- p.log.error(`Error in exclude pattern: ${excludeError}`);
291
+ logger.error(`Error in exclude pattern: ${excludeError}`);
280
292
  process.exit(1);
281
293
  }
282
294
  }
283
295
  const depthStr = args.includes("--single-page") ? "0" : getArgValue("--depth") || getArgValue("-d") || "3";
284
296
  const depth = Number(depthStr);
285
297
  if (!Number.isInteger(depth) || depth < 0 || depth > 10) {
286
- p.log.error("Error: Depth must be an integer between 0 and 10");
298
+ logger.error("Error: Depth must be an integer between 0 and 10");
287
299
  process.exit(1);
288
300
  }
289
301
  const driver = getArgValue("--driver");
290
302
  if (driver && driver !== "http" && driver !== "playwright") {
291
- p.log.error("Error: Driver must be either \"http\" or \"playwright\"");
303
+ logger.error("Error: Driver must be either \"http\" or \"playwright\"");
292
304
  process.exit(1);
293
305
  }
294
306
  const maxPagesStr = getArgValue("--max-pages");
295
307
  if (maxPagesStr) {
296
308
  const maxPages = Number.parseInt(maxPagesStr);
297
309
  if (Number.isNaN(maxPages) || maxPages < 1) {
298
- p.log.error("Error: Max pages must be a positive number");
310
+ logger.error("Error: Max pages must be a positive number");
299
311
  process.exit(1);
300
312
  }
301
313
  }
@@ -303,7 +315,7 @@ Examples:
303
315
  if (crawlDelayStr) {
304
316
  const crawlDelay = Number.parseInt(crawlDelayStr);
305
317
  if (Number.isNaN(crawlDelay) || crawlDelay < 0) {
306
- p.log.error("Error: Crawl delay must be a non-negative number");
318
+ logger.error("Error: Crawl delay must be a non-negative number");
307
319
  process.exit(1);
308
320
  }
309
321
  }
@@ -319,7 +331,7 @@ Examples:
319
331
  "markdown"
320
332
  ];
321
333
  for (const artifact of artifacts) if (!validArtifacts.includes(artifact)) {
322
- p.log.error(`Error: Invalid artifact '${artifact}'. Valid options: ${validArtifacts.join(", ")}`);
334
+ logger.error(`Error: Invalid artifact '${artifact}'. Valid options: ${validArtifacts.join(", ")}`);
323
335
  process.exit(1);
324
336
  }
325
337
  const originOverride = getArgValue("--origin");
@@ -338,7 +350,7 @@ Examples:
338
350
  const verbose = args.includes("--verbose") || args.includes("-v");
339
351
  const skipSitemap = args.includes("--skip-sitemap");
340
352
  const allowSubdomains = args.includes("--allow-subdomains");
341
- if (skipSitemap && parsed.isGlob) p.log.warn("Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.");
353
+ if (skipSitemap && parsed.isGlob) logger.warn("Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.");
342
354
  return {
343
355
  urls: [url],
344
356
  outputDir: resolve(getArgValue("--output") || getArgValue("-o") || "output"),
@@ -357,12 +369,16 @@ Examples:
357
369
  exclude: excludePatterns.length > 0 ? excludePatterns : void 0,
358
370
  verbose,
359
371
  skipSitemap,
360
- allowSubdomains
372
+ allowSubdomains,
373
+ silent
361
374
  };
362
375
  }
363
376
  async function main() {
364
377
  const cliOptions = parseCliArgs();
365
378
  const fileConfig = await loadMdreamConfig();
379
+ const silent = (cliOptions?.silent ?? false) || (fileConfig.silent ?? false);
380
+ const logger = resolveLogger({ silent });
381
+ runLogger = logger;
366
382
  let options;
367
383
  if (cliOptions) {
368
384
  const configExclude = fileConfig.exclude || [];
@@ -375,10 +391,11 @@ async function main() {
375
391
  skipSitemap: cliOptions.skipSitemap || fileConfig.skipSitemap || false,
376
392
  allowSubdomains: cliOptions.allowSubdomains || fileConfig.allowSubdomains || false,
377
393
  verbose: cliOptions.verbose || fileConfig.verbose || false,
394
+ silent,
378
395
  exclude: configExclude.length > 0 || cliExclude.length > 0 ? [...configExclude, ...cliExclude] : void 0,
379
396
  hooks: fileConfig.hooks
380
397
  };
381
- p.intro(`☁️ mdream v${version}`);
398
+ logger.intro(`☁️ mdream v${version}`);
382
399
  const formats = [];
383
400
  if (options.generateLlmsTxt) formats.push("llms.txt");
384
401
  if (options.generateLlmsFullTxt) formats.push("llms-full.txt");
@@ -393,35 +410,35 @@ async function main() {
393
410
  options.allowSubdomains && `Allow subdomains: Yes`,
394
411
  options.verbose && `Verbose: Enabled`
395
412
  ].filter(Boolean);
396
- p.note(summary.join("\n"), "Configuration");
413
+ logger.note(summary.join("\n"), "Configuration");
397
414
  } else options = await interactiveCrawl();
398
415
  if (!options) process.exit(0);
399
416
  const permCheck = checkOutputDirectoryPermissions(options.outputDir);
400
417
  if (!permCheck.success) {
401
- p.log.error(permCheck.error);
402
- if (permCheck.error?.includes("Permission denied")) p.log.info("Tip: Try running with elevated privileges (e.g., sudo) or change the output directory permissions.");
418
+ logger.error(permCheck.error);
419
+ if (permCheck.error?.includes("Permission denied")) logger.info("Tip: Try running with elevated privileges (e.g., sudo) or change the output directory permissions.");
403
420
  process.exit(1);
404
421
  }
405
422
  if (options.driver === "playwright") {
406
423
  try {
407
424
  await import("crawlee");
408
425
  } catch {
409
- p.log.error("The Playwright driver requires crawlee. Install it with: npm install crawlee");
426
+ logger.error("The Playwright driver requires crawlee. Install it with: npm install crawlee");
410
427
  process.exit(1);
411
428
  }
412
429
  const { ensurePlaywrightInstalled, isUseChromeSupported } = await import("./_chunks/playwright-utils.mjs");
413
430
  if (await isUseChromeSupported()) {
414
431
  options.useChrome = true;
415
- p.log.info("System Chrome detected and enabled.");
432
+ logger.info("System Chrome detected and enabled.");
416
433
  } else {
417
- if (!await ensurePlaywrightInstalled()) {
418
- p.log.error("Cannot proceed without Playwright. Please install it manually or use the HTTP driver instead.");
434
+ if (!await ensurePlaywrightInstalled(logger)) {
435
+ logger.error("Cannot proceed without Playwright. Please install it manually or use the HTTP driver instead.");
419
436
  process.exit(1);
420
437
  }
421
- p.log.info("Using global playwright instance.");
438
+ logger.info("Using global playwright instance.");
422
439
  }
423
440
  }
424
- const s = p.spinner();
441
+ const s = logger.spinner();
425
442
  s.start("Discovering sitemaps");
426
443
  const startTime = Date.now();
427
444
  let crawlStartTime = 0;
@@ -449,9 +466,9 @@ async function main() {
449
466
  const failed = results.filter((r) => !r.success).length;
450
467
  const failedResults = results.filter((r) => !r.success);
451
468
  if (failed > 0 && cliOptions) {
452
- p.log.error("Failed URLs:");
469
+ logger.error("Failed URLs:");
453
470
  failedResults.forEach((result) => {
454
- p.log.error(` ${result.url}: ${result.error || "Unknown error"}`);
471
+ logger.error(` ${result.url}: ${result.error || "Unknown error"}`);
455
472
  });
456
473
  } else if (failed > 0) {
457
474
  console.log("\nFailed URLs:");
@@ -465,13 +482,14 @@ async function main() {
465
482
  if (options.generateLlmsFullTxt) generatedFiles.push("llms-full.txt");
466
483
  if (options.generateIndividualMd) generatedFiles.push(`${successful} MD files`);
467
484
  }
468
- await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationSeconds, lastProgress?.crawling.latency);
485
+ await showCrawlResults(logger, successful, failed, options.outputDir, generatedFiles, durationSeconds, lastProgress?.crawling.latency);
469
486
  process.exit(0);
470
487
  }
471
488
  main().catch((error) => {
489
+ const logger = runLogger;
472
490
  const msg = error instanceof Error ? error.message : String(error);
473
- if (msg.includes("wmic") || msg.includes("ENOENT") && process.platform === "win32") p.log.error("Crawlee failed because wmic.exe is not available on this system. Windows 11 removed wmic.exe, which older crawlee versions depend on for memory monitoring.\nFix: upgrade crawlee to >=3.16.0 or switch to the HTTP driver (--driver http).");
474
- else p.log.error(`Unexpected error: ${msg}`);
491
+ if (msg.includes("wmic") || msg.includes("ENOENT") && process.platform === "win32") logger.error("Crawlee failed because wmic.exe is not available on this system. Windows 11 removed wmic.exe, which older crawlee versions depend on for memory monitoring.\nFix: upgrade crawlee to >=3.16.0 or switch to the HTTP driver (--driver http).");
492
+ else logger.error(`Unexpected error: ${msg}`);
475
493
  process.exit(1);
476
494
  });
477
495
  export {};
package/dist/index.d.mts CHANGED
@@ -1,3 +1,52 @@
1
+ /**
2
+ * Minimal spinner surface used by the crawler. Mirrors the subset of
3
+ * `@clack/prompts` spinner methods the package relies on.
4
+ */
5
+ interface CrawlSpinner {
6
+ start: (message?: string) => void;
7
+ message: (message?: string) => void;
8
+ stop: (message?: string, code?: number) => void;
9
+ }
10
+ /**
11
+ * Diagnostic/progress sink for the crawler. All human-facing, non-data output
12
+ * (the messages that would otherwise pollute stdout) flows through this single
13
+ * seam so it can be silenced or redirected. It deliberately excludes the
14
+ * interactive prompts (`select`, `text`, `confirm`, ...) which are input
15
+ * control flow, not log output, and only run in the interactive CLI.
16
+ *
17
+ * Provide a custom implementation via `CrawlOptions.logger` to route messages
18
+ * anywhere (e.g. stderr for an MCP server), or set `CrawlOptions.silent` to
19
+ * drop them entirely. See issue #100.
20
+ */
21
+ interface CrawlLogger {
22
+ intro: (title?: string) => void;
23
+ note: (message: string, title?: string) => void;
24
+ cancel: (message?: string) => void;
25
+ info: (message: string) => void;
26
+ warn: (message: string) => void;
27
+ error: (message: string) => void;
28
+ success: (message: string) => void;
29
+ spinner: () => CrawlSpinner;
30
+ }
31
+ /**
32
+ * A logger that drops every message. Used when `silent` is set so a library
33
+ * consumer (e.g. an MCP server) gets a clean stdout.
34
+ */
35
+ declare const silentLogger: CrawlLogger;
36
+ /**
37
+ * The default logger, backed by `@clack/prompts` (writes to stdout). Preserves
38
+ * the existing CLI output exactly.
39
+ */
40
+ declare function createClackLogger(): CrawlLogger;
41
+ /**
42
+ * Resolve the effective logger from crawl options. An explicit `logger` wins;
43
+ * otherwise `silent` selects the no-op sink, and the default is the clack
44
+ * logger. Pure: same inputs, same logger.
45
+ */
46
+ declare function resolveLogger(options: {
47
+ silent?: boolean;
48
+ logger?: CrawlLogger;
49
+ }): CrawlLogger;
1
50
  interface PageData {
2
51
  url: string;
3
52
  html: string;
@@ -47,6 +96,17 @@ interface CrawlOptions {
47
96
  verbose?: boolean;
48
97
  skipSitemap?: boolean;
49
98
  allowSubdomains?: boolean;
99
+ /**
100
+ * Suppress all diagnostic/progress logging. Use when stdout must stay clean,
101
+ * e.g. an MCP server that only emits JSON-RPC (issue #100). Ignored when an
102
+ * explicit `logger` is provided.
103
+ */
104
+ silent?: boolean;
105
+ /**
106
+ * Custom sink for diagnostic/progress messages. Route logs anywhere (e.g.
107
+ * stderr) instead of the default `@clack/prompts` stdout output.
108
+ */
109
+ logger?: CrawlLogger;
50
110
  hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][] }>;
51
111
  onPage?: (page: PageData) => Promise<void> | void;
52
112
  }
@@ -59,6 +119,8 @@ interface MdreamCrawlConfig {
59
119
  skipSitemap?: boolean;
60
120
  allowSubdomains?: boolean;
61
121
  verbose?: boolean;
122
+ /** Suppress all diagnostic/progress logging (issue #100). */
123
+ silent?: boolean;
62
124
  artifacts?: ('llms.txt' | 'llms-full.txt' | 'markdown')[];
63
125
  hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][] }>;
64
126
  }
@@ -111,4 +173,4 @@ interface CrawlProgress {
111
173
  };
112
174
  }
113
175
  declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
114
- export { type CrawlHooks, type CrawlOptions, type CrawlResult, type MdreamCrawlConfig, type PageData, crawlAndGenerate, defineConfig };
176
+ export { type CrawlHooks, type CrawlLogger, type CrawlOptions, type CrawlResult, type CrawlSpinner, type MdreamCrawlConfig, type PageData, crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger };
package/dist/index.mjs CHANGED
@@ -1,5 +1,5 @@
1
- import { t as crawlAndGenerate } from "./_chunks/crawl.mjs";
1
+ import { crawlAndGenerate, createClackLogger, resolveLogger, silentLogger } from "./_chunks/crawl.mjs";
2
2
  function defineConfig(config) {
3
3
  return config;
4
4
  }
5
- export { crawlAndGenerate, defineConfig };
5
+ export { crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mdream/crawl",
3
3
  "type": "module",
4
- "version": "1.2.1",
4
+ "version": "1.3.0",
5
5
  "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -54,17 +54,17 @@
54
54
  }
55
55
  },
56
56
  "dependencies": {
57
- "@clack/prompts": "^1.4.0",
57
+ "@clack/prompts": "^1.5.1",
58
58
  "c12": "^3.3.4",
59
59
  "hookable": "^6.1.1",
60
60
  "nypm": "^0.6.6",
61
61
  "ofetch": "^1.5.1",
62
62
  "pathe": "^2.0.3",
63
63
  "picomatch": "^4.0.4",
64
- "tldts": "^7.0.30",
64
+ "tldts": "^7.4.2",
65
65
  "ufo": "^1.6.4",
66
- "@mdream/js": "1.2.1",
67
- "mdream": "1.2.1"
66
+ "@mdream/js": "1.3.0",
67
+ "mdream": "1.3.0"
68
68
  },
69
69
  "devDependencies": {
70
70
  "@types/picomatch": "^4.0.3"