@mdream/crawl 1.7.1 → 2.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -184,7 +184,10 @@ function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
184
184
  if (parsedPattern.isGlob) return matchesGlobPattern(url, parsedPattern, allowSubdomains);
185
185
  return url === pattern;
186
186
  }
187
- if (pattern.startsWith("/")) return picomatch(pattern.endsWith("/*") ? pattern.replace("/*", "/**") : pattern)(urlPath);
187
+ if (pattern.startsWith("/")) {
188
+ const adjustedPattern = pattern.endsWith("/*") ? pattern.replace("/*", "/**") : pattern;
189
+ return picomatch(adjustedPattern)(urlPath);
190
+ }
188
191
  return picomatch(pattern)(urlPath) || picomatch(pattern)(urlPath.substring(1));
189
192
  });
190
193
  } catch {
@@ -651,7 +654,8 @@ async function crawlAndGenerate(options, onProgress) {
651
654
  const hostPrefix = allowSubdomains ? [parsedUrl.hostname.replace(URL_PATH_UNSAFE_CHARS_RE, "-")] : [];
652
655
  const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
653
656
  const safeSegments = [...hostPrefix, ...pathSegments.map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"))];
654
- const safeFilename = normalize(`${safeSegments.length > 0 ? safeSegments.join("/") : "index"}.md`);
657
+ const filename = safeSegments.length > 0 ? safeSegments.join("/") : "index";
658
+ const safeFilename = normalize(`${filename}.md`);
655
659
  filePath = join(outputDir, safeFilename);
656
660
  }
657
661
  const isHomePage = parsedUrl.pathname === "/" && parsedUrl.origin === normalizedHomePageUrl;
@@ -885,18 +889,19 @@ async function crawlAndGenerate(options, onProgress) {
885
889
  return result.content.trim().replace(FRONTMATTER_BLOCK_RE, "").trim().length > 10;
886
890
  });
887
891
  const seenUrls = /* @__PURE__ */ new Set();
892
+ const processedFiles = contentResults.filter((result) => {
893
+ if (seenUrls.has(result.url)) return false;
894
+ seenUrls.add(result.url);
895
+ return true;
896
+ }).map((result) => ({
897
+ filePath: result.filePath,
898
+ title: result.title,
899
+ content: result.content,
900
+ url: result.url,
901
+ metadata: result.metadata
902
+ }));
888
903
  const llmsResult = await generateLlmsTxtArtifacts({
889
- files: contentResults.filter((result) => {
890
- if (seenUrls.has(result.url)) return false;
891
- seenUrls.add(result.url);
892
- return true;
893
- }).map((result) => ({
894
- filePath: result.filePath,
895
- title: result.title,
896
- content: result.content,
897
- url: result.url,
898
- metadata: result.metadata
899
- })),
904
+ files: processedFiles,
900
905
  siteName,
901
906
  description,
902
907
  origin: originUrl || firstUrl.origin,
package/dist/cli.d.mts CHANGED
@@ -1 +1 @@
1
- export { };
1
+ export {}
package/dist/cli.mjs CHANGED
@@ -21,7 +21,8 @@ function isSilentArgv(args) {
21
21
  return args.some((arg) => QUIET_FLAGS.has(arg));
22
22
  }
23
23
  let runLogger = resolveLogger({ silent: isSilentArgv(process.argv.slice(2)) });
24
- const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
24
+ const __dirname = dirname(fileURLToPath(import.meta.url));
25
+ const packageJsonPath = join(__dirname, "..", "package.json");
25
26
  const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
26
27
  function checkOutputDirectoryPermissions(outputDir) {
27
28
  try {
@@ -79,7 +80,7 @@ async function interactiveCrawl() {
79
80
  }
80
81
  }
81
82
  });
82
- if (p.isCancel(urlsInput)) {
83
+ if (typeof urlsInput !== "string") {
83
84
  p.cancel("Operation cancelled.");
84
85
  return null;
85
86
  }
package/dist/index.d.mts CHANGED
@@ -28,7 +28,7 @@ interface BoilerplateOptions {
28
28
  * Returns a new array aligned with `contents`. Documents that are unchanged (no
29
29
  * detected chrome, or detection skipped) are returned byte-for-byte as given.
30
30
  */
31
- declare function stripBoilerplateFromCorpus(contents: string[], options?: BoilerplateOptions): string[];
31
+ export declare function stripBoilerplateFromCorpus(contents: string[], options?: BoilerplateOptions): string[];
32
32
  /**
33
33
  * Minimal spinner surface used by the crawler. Mirrors the subset of
34
34
  * `@clack/prompts` spinner methods the package relies on.
@@ -63,18 +63,18 @@ interface CrawlLogger {
63
63
  * A logger that drops every message. Used when `silent` is set so a library
64
64
  * consumer (e.g. an MCP server) gets a clean stdout.
65
65
  */
66
- declare const silentLogger: CrawlLogger;
66
+ export declare const silentLogger: CrawlLogger;
67
67
  /**
68
68
  * The default logger, backed by `@clack/prompts` (writes to stdout). Preserves
69
69
  * the existing CLI output exactly.
70
70
  */
71
- declare function createClackLogger(): CrawlLogger;
71
+ export declare function createClackLogger(): CrawlLogger;
72
72
  /**
73
73
  * Resolve the effective logger from crawl options. An explicit `logger` wins;
74
74
  * otherwise `silent` selects the no-op sink, and the default is the clack
75
75
  * logger. Pure: same inputs, same logger.
76
76
  */
77
- declare function resolveLogger(options: {
77
+ export declare function resolveLogger(options: {
78
78
  silent?: boolean;
79
79
  logger?: CrawlLogger;
80
80
  }): CrawlLogger;
@@ -182,7 +182,7 @@ interface MdreamCrawlConfig {
182
182
  artifacts?: ('llms.txt' | 'llms-full.txt' | 'markdown')[];
183
183
  hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
184
184
  }
185
- declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
185
+ export declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
186
186
  interface ParsedUrlPattern {
187
187
  baseUrl: string;
188
188
  pattern: string;
@@ -231,5 +231,5 @@ interface CrawlProgress {
231
231
  current?: string;
232
232
  };
233
233
  }
234
- declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
235
- export { type BoilerplateOptions, type CrawlHooks, type CrawlLogger, type CrawlOptions, type CrawlResult, type CrawlSpinner, type MdreamCrawlConfig, type PageData, crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger, stripBoilerplateFromCorpus };
234
+ export declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
235
+ export type { BoilerplateOptions, CrawlHooks, CrawlLogger, CrawlOptions, CrawlResult, CrawlSpinner, MdreamCrawlConfig, PageData };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mdream/crawl",
3
3
  "type": "module",
4
- "version": "1.7.1",
4
+ "version": "2.0.0-beta.1",
5
5
  "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -54,17 +54,17 @@
54
54
  }
55
55
  },
56
56
  "dependencies": {
57
- "@clack/prompts": "^1.7.0",
58
- "c12": "^3.3.4",
59
- "hookable": "^6.1.1",
60
- "nypm": "^0.6.9",
57
+ "@clack/prompts": "^1.8.1",
58
+ "@mdream/js": "2.0.0-beta.1",
59
+ "c12": "4.0.0-rc.2",
60
+ "hookable": "^6.1.2",
61
+ "mdream": "2.0.0-beta.1",
62
+ "nypm": "^0.6.10",
61
63
  "ofetch": "^1.5.1",
62
64
  "pathe": "^2.0.3",
63
- "picomatch": "^4.0.5",
64
- "tldts": "^7.4.10",
65
- "ufo": "^1.6.4",
66
- "@mdream/js": "1.7.1",
67
- "mdream": "1.7.1"
65
+ "picomatch": "^4.0.7",
66
+ "tldts": "^7.4.16",
67
+ "ufo": "^1.6.4"
68
68
  },
69
69
  "devDependencies": {
70
70
  "@types/picomatch": "^4.0.3"