@mdream/crawl 2.0.0-beta.1 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -78,6 +78,10 @@ export declare function resolveLogger(options: {
78
78
  silent?: boolean;
79
79
  logger?: CrawlLogger;
80
80
  }): CrawlLogger;
81
+ /** An output file the crawler can write. */
82
+ type CrawlArtifact = 'llms.txt' | 'llms-full.txt' | 'markdown';
83
+ /** How the crawler fetches pages. */
84
+ type CrawlDriver = 'http' | 'playwright';
81
85
  interface PageData {
82
86
  url: string;
83
87
  html: string;
@@ -106,83 +110,79 @@ interface CrawlHooks {
106
110
  results: CrawlResult[];
107
111
  }) => void | Promise<void>;
108
112
  }
113
+ /**
114
+ * Options for `crawlAndGenerate`, the `mdream.config.*` file, and the CLI.
115
+ * The CLI flag for each option is its kebab-case name (`maxPages` is `--max-pages`).
116
+ */
109
117
  interface CrawlOptions {
118
+ /** Starting URLs. Glob patterns such as `example.com/docs/**` limit the crawl. */
110
119
  urls: string[];
111
- outputDir: string;
112
- maxRequestsPerCrawl?: number;
113
- generateLlmsTxt?: boolean;
114
- generateLlmsFullTxt?: boolean;
115
- generateIndividualMd?: boolean;
120
+ /** Output directory. Defaults to `output`. */
121
+ output?: string;
122
+ /**
123
+ * Link depth. `0` processes only the given URLs: no sitemap or robots.txt
124
+ * discovery and no link following. `1` or more discovers the sitemap. If the
125
+ * sitemap gives no URLs, the crawler follows links up to `depth` hops from
126
+ * the starting URLs. Integer from 0 to 10. Defaults to `3`.
127
+ */
128
+ depth?: number;
129
+ /** Maximum number of pages to fetch. Defaults to no limit. */
130
+ maxPages?: number;
131
+ /**
132
+ * Sitemap URL or URLs to use instead of auto-discovery. All parts are loaded
133
+ * and merged. Ignored when `skipSitemap` is set.
134
+ */
135
+ sitemap?: string | string[];
136
+ /** Files to write. Defaults to all three. An empty array writes no files. */
137
+ artifacts?: CrawlArtifact[];
138
+ /** Site name in llms.txt. Defaults to the home page title. */
139
+ siteName?: string;
140
+ /** Site description in llms.txt. Defaults to the home page description. */
141
+ description?: string;
142
+ /** Origin for relative links in page Markdown. Defaults to each page's own origin. */
116
143
  origin?: string;
117
- chunkSize?: number;
118
- driver?: 'http' | 'playwright';
144
+ /** Defaults to `http`. */
145
+ driver?: CrawlDriver;
146
+ /** Use the system Chrome with the Playwright driver. Defaults to `false`. */
119
147
  useChrome?: boolean;
120
- followLinks?: boolean;
121
- maxDepth?: number;
122
- globPatterns?: ParsedUrlPattern[];
123
- crawlDelay?: number;
148
+ /** Glob patterns for URLs to skip. */
124
149
  exclude?: string[];
125
- siteNameOverride?: string;
126
- descriptionOverride?: string;
127
- verbose?: boolean;
150
+ /** Delay between requests in seconds. Defaults to the robots.txt `Crawl-delay`. */
151
+ crawlDelay?: number;
152
+ /** Skip sitemap and robots.txt discovery. Defaults to `false`. */
128
153
  skipSitemap?: boolean;
129
- /**
130
- * Explicit sitemap URL(s) to use instead of auto-discovery (robots.txt +
131
- * well-known paths). Accepts non-standard locations and multiple parts, which
132
- * are all loaded and merged. Ignored when `skipSitemap` is set.
133
- */
134
- sitemapUrls?: string[];
154
+ /** Crawl other subdomains of the same registrable domain. Defaults to `false`. */
135
155
  allowSubdomains?: boolean;
136
156
  /**
137
- * Strip repeated site chrome (top nav, footer, newsletter walls) from per-page
138
- * markdown and llms-full.txt page sections. Implemented as a post-processing
139
- * pass over the generated markdown: blocks that repeat across the corpus are
140
- * detected by frequency and removed only from the wrapping (leading/trailing)
141
- * run of each page, so a page's main content is preserved. Does not affect the
142
- * llms.txt link index. Defaults to true. Needs a corpus (>= 3 pages) to run, so
143
- * single-page crawls are left unchanged.
157
+ * Remove repeated site chrome (navigation, footers) from page Markdown and
158
+ * llms-full.txt. Blocks that repeat across the corpus are removed only from
159
+ * the start and end of each page. The llms.txt link index does not change.
160
+ * Needs at least 3 pages. Defaults to `true`.
144
161
  */
145
162
  stripBoilerplate?: boolean;
146
163
  /**
147
- * Fraction of crawled pages a block must appear in to count as chrome (0..1).
148
- * Higher is stricter. Defaults to `DEFAULT_BOILERPLATE_THRESHOLD` (0.5).
164
+ * Fraction of pages a block must appear in to count as chrome. Greater than
165
+ * 0 and at most 1. Higher is stricter. Defaults to `0.5`.
149
166
  */
150
167
  boilerplateThreshold?: number;
168
+ /** Log errors for each failed URL. Defaults to `false`. */
169
+ verbose?: boolean;
151
170
  /**
152
- * Suppress all diagnostic/progress logging. Use when stdout must stay clean,
153
- * e.g. an MCP server that only emits JSON-RPC (issue #100). Ignored when an
154
- * explicit `logger` is provided.
171
+ * Drop all diagnostic and progress logs. Use it when stdout must stay clean,
172
+ * for example in an MCP server. An explicit `logger` takes precedence.
155
173
  */
156
174
  silent?: boolean;
157
- /**
158
- * Custom sink for diagnostic/progress messages. Route logs anywhere (e.g.
159
- * stderr) instead of the default `@clack/prompts` stdout output.
160
- */
175
+ /** Sink for diagnostic and progress logs. Defaults to `@clack/prompts` on stdout. */
161
176
  logger?: CrawlLogger;
162
177
  hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
163
- onPage?: (page: PageData) => Promise<void> | void;
164
- }
165
- interface MdreamCrawlConfig {
166
- exclude?: string[];
167
- driver?: 'http' | 'playwright';
168
- maxDepth?: number;
169
- maxPages?: number;
170
- crawlDelay?: number;
171
- skipSitemap?: boolean;
172
- /** Explicit sitemap URL(s) to use instead of auto-discovery. Supports non-standard locations and multiple parts. */
173
- sitemap?: string | string[];
174
- allowSubdomains?: boolean;
175
- /** Strip repeated site chrome from per-page output. Defaults to true. */
176
- stripBoilerplate?: boolean;
177
- /** Fraction of pages a block must repeat in to count as chrome (0..1). Defaults to 0.5. */
178
- boilerplateThreshold?: number;
179
- verbose?: boolean;
180
- /** Suppress all diagnostic/progress logging (issue #100). */
181
- silent?: boolean;
182
- artifacts?: ('llms.txt' | 'llms-full.txt' | 'markdown')[];
183
- hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
178
+ /**
179
+ * Parsed URL patterns that replace the patterns parsed from `urls`.
180
+ * Advanced: `urls` covers most needs.
181
+ */
182
+ globPatterns?: ParsedUrlPattern[];
184
183
  }
185
- export declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
184
+ /** Type helper for `mdream.config.*`. Every option is optional in the config file. */
185
+ export declare function defineConfig(config: Partial<CrawlOptions>): Partial<CrawlOptions>;
186
186
  interface ParsedUrlPattern {
187
187
  baseUrl: string;
188
188
  pattern: string;
@@ -231,5 +231,10 @@ interface CrawlProgress {
231
231
  current?: string;
232
232
  };
233
233
  }
234
+ /**
235
+ * Crawl a site and write the requested artifacts. Options are parsed once with
236
+ * `resolveCrawlOptions`; an unknown key or an invalid value throws a `TypeError`
237
+ * before any request is sent.
238
+ */
234
239
  export declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
235
- export type { BoilerplateOptions, CrawlHooks, CrawlLogger, CrawlOptions, CrawlResult, CrawlSpinner, MdreamCrawlConfig, PageData };
240
+ export type { BoilerplateOptions, CrawlArtifact, CrawlDriver, CrawlHooks, CrawlLogger, CrawlOptions, CrawlProgress, CrawlResult, CrawlSpinner, PageData, PageMetadata, ParsedUrlPattern };
package/package.json CHANGED
@@ -1,13 +1,23 @@
1
1
  {
2
2
  "name": "@mdream/crawl",
3
3
  "type": "module",
4
- "version": "2.0.0-beta.1",
4
+ "version": "2.0.1",
5
5
  "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
8
8
  "email": "harlan@harlanzw.com",
9
9
  "url": "https://harlanzw.com/"
10
10
  },
11
+ "contributors": [
12
+ {
13
+ "name": "André Cruz",
14
+ "url": "https://github.com/edevil"
15
+ },
16
+ {
17
+ "name": "Bernardo Pereira",
18
+ "url": "https://github.com/BernardoPe"
19
+ }
20
+ ],
11
21
  "license": "MIT",
12
22
  "repository": {
13
23
  "type": "git",
@@ -35,7 +45,7 @@
35
45
  "main": "./dist/index.mjs",
36
46
  "types": "./dist/index.d.mts",
37
47
  "bin": {
38
- "@mdream/crawl": "./bin/mdream-crawl.mjs"
48
+ "mdream-crawl": "./bin/mdream-crawl.mjs"
39
49
  },
40
50
  "files": [
41
51
  "bin",
@@ -55,15 +65,15 @@
55
65
  },
56
66
  "dependencies": {
57
67
  "@clack/prompts": "^1.8.1",
58
- "@mdream/js": "2.0.0-beta.1",
68
+ "@mdream/js": "2.0.1",
59
69
  "c12": "4.0.0-rc.2",
60
70
  "hookable": "^6.1.2",
61
- "mdream": "2.0.0-beta.1",
71
+ "mdream": "2.0.1",
62
72
  "nypm": "^0.6.10",
63
73
  "ofetch": "^1.5.1",
64
74
  "pathe": "^2.0.3",
65
75
  "picomatch": "^4.0.7",
66
- "tldts": "^7.4.16",
76
+ "tldts": "^7.4.18",
67
77
  "ufo": "^1.6.4"
68
78
  },
69
79
  "devDependencies": {