@mdream/crawl 1.7.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +194 -88
- package/dist/_chunks/crawl.mjs +237 -29
- package/dist/cli.mjs +329 -359
- package/dist/index.d.mts +64 -59
- package/package.json +19 -9
package/dist/index.d.mts
CHANGED
|
@@ -78,6 +78,10 @@ export declare function resolveLogger(options: {
|
|
|
78
78
|
silent?: boolean;
|
|
79
79
|
logger?: CrawlLogger;
|
|
80
80
|
}): CrawlLogger;
|
|
81
|
+
/** An output file the crawler can write. */
|
|
82
|
+
type CrawlArtifact = 'llms.txt' | 'llms-full.txt' | 'markdown';
|
|
83
|
+
/** How the crawler fetches pages. */
|
|
84
|
+
type CrawlDriver = 'http' | 'playwright';
|
|
81
85
|
interface PageData {
|
|
82
86
|
url: string;
|
|
83
87
|
html: string;
|
|
@@ -106,83 +110,79 @@ interface CrawlHooks {
|
|
|
106
110
|
results: CrawlResult[];
|
|
107
111
|
}) => void | Promise<void>;
|
|
108
112
|
}
|
|
113
|
+
/**
|
|
114
|
+
* Options for `crawlAndGenerate`, the `mdream.config.*` file, and the CLI.
|
|
115
|
+
* The CLI flag for each option is its kebab-case name (`maxPages` is `--max-pages`).
|
|
116
|
+
*/
|
|
109
117
|
interface CrawlOptions {
|
|
118
|
+
/** Starting URLs. Glob patterns such as `example.com/docs/**` limit the crawl. */
|
|
110
119
|
urls: string[];
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
120
|
+
/** Output directory. Defaults to `output`. */
|
|
121
|
+
output?: string;
|
|
122
|
+
/**
|
|
123
|
+
* Link depth. `0` processes only the given URLs: no sitemap or robots.txt
|
|
124
|
+
* discovery and no link following. `1` or more discovers the sitemap. If the
|
|
125
|
+
* sitemap gives no URLs, the crawler follows links up to `depth` hops from
|
|
126
|
+
* the starting URLs. Integer from 0 to 10. Defaults to `3`.
|
|
127
|
+
*/
|
|
128
|
+
depth?: number;
|
|
129
|
+
/** Maximum number of pages to fetch. Defaults to no limit. */
|
|
130
|
+
maxPages?: number;
|
|
131
|
+
/**
|
|
132
|
+
* Sitemap URL or URLs to use instead of auto-discovery. All parts are loaded
|
|
133
|
+
* and merged. Ignored when `skipSitemap` is set.
|
|
134
|
+
*/
|
|
135
|
+
sitemap?: string | string[];
|
|
136
|
+
/** Files to write. Defaults to all three. An empty array writes no files. */
|
|
137
|
+
artifacts?: CrawlArtifact[];
|
|
138
|
+
/** Site name in llms.txt. Defaults to the home page title. */
|
|
139
|
+
siteName?: string;
|
|
140
|
+
/** Site description in llms.txt. Defaults to the home page description. */
|
|
141
|
+
description?: string;
|
|
142
|
+
/** Origin for relative links in page Markdown. Defaults to each page's own origin. */
|
|
116
143
|
origin?: string;
|
|
117
|
-
|
|
118
|
-
driver?:
|
|
144
|
+
/** Defaults to `http`. */
|
|
145
|
+
driver?: CrawlDriver;
|
|
146
|
+
/** Use the system Chrome with the Playwright driver. Defaults to `false`. */
|
|
119
147
|
useChrome?: boolean;
|
|
120
|
-
|
|
121
|
-
maxDepth?: number;
|
|
122
|
-
globPatterns?: ParsedUrlPattern[];
|
|
123
|
-
crawlDelay?: number;
|
|
148
|
+
/** Glob patterns for URLs to skip. */
|
|
124
149
|
exclude?: string[];
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
150
|
+
/** Delay between requests in seconds. Defaults to the robots.txt `Crawl-delay`. */
|
|
151
|
+
crawlDelay?: number;
|
|
152
|
+
/** Skip sitemap and robots.txt discovery. Defaults to `false`. */
|
|
128
153
|
skipSitemap?: boolean;
|
|
129
|
-
/**
|
|
130
|
-
* Explicit sitemap URL(s) to use instead of auto-discovery (robots.txt +
|
|
131
|
-
* well-known paths). Accepts non-standard locations and multiple parts, which
|
|
132
|
-
* are all loaded and merged. Ignored when `skipSitemap` is set.
|
|
133
|
-
*/
|
|
134
|
-
sitemapUrls?: string[];
|
|
154
|
+
/** Crawl other subdomains of the same registrable domain. Defaults to `false`. */
|
|
135
155
|
allowSubdomains?: boolean;
|
|
136
156
|
/**
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
* run of each page, so a page's main content is preserved. Does not affect the
|
|
142
|
-
* llms.txt link index. Defaults to true. Needs a corpus (>= 3 pages) to run, so
|
|
143
|
-
* single-page crawls are left unchanged.
|
|
157
|
+
* Remove repeated site chrome (navigation, footers) from page Markdown and
|
|
158
|
+
* llms-full.txt. Blocks that repeat across the corpus are removed only from
|
|
159
|
+
* the start and end of each page. The llms.txt link index does not change.
|
|
160
|
+
* Needs at least 3 pages. Defaults to `true`.
|
|
144
161
|
*/
|
|
145
162
|
stripBoilerplate?: boolean;
|
|
146
163
|
/**
|
|
147
|
-
* Fraction of
|
|
148
|
-
* Higher is stricter. Defaults to `
|
|
164
|
+
* Fraction of pages a block must appear in to count as chrome. Greater than
|
|
165
|
+
* 0 and at most 1. Higher is stricter. Defaults to `0.5`.
|
|
149
166
|
*/
|
|
150
167
|
boilerplateThreshold?: number;
|
|
168
|
+
/** Log errors for each failed URL. Defaults to `false`. */
|
|
169
|
+
verbose?: boolean;
|
|
151
170
|
/**
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
* explicit `logger` is provided.
|
|
171
|
+
* Drop all diagnostic and progress logs. Use it when stdout must stay clean,
|
|
172
|
+
* for example in an MCP server. An explicit `logger` takes precedence.
|
|
155
173
|
*/
|
|
156
174
|
silent?: boolean;
|
|
157
|
-
/**
|
|
158
|
-
* Custom sink for diagnostic/progress messages. Route logs anywhere (e.g.
|
|
159
|
-
* stderr) instead of the default `@clack/prompts` stdout output.
|
|
160
|
-
*/
|
|
175
|
+
/** Sink for diagnostic and progress logs. Defaults to `@clack/prompts` on stdout. */
|
|
161
176
|
logger?: CrawlLogger;
|
|
162
177
|
hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
maxDepth?: number;
|
|
169
|
-
maxPages?: number;
|
|
170
|
-
crawlDelay?: number;
|
|
171
|
-
skipSitemap?: boolean;
|
|
172
|
-
/** Explicit sitemap URL(s) to use instead of auto-discovery. Supports non-standard locations and multiple parts. */
|
|
173
|
-
sitemap?: string | string[];
|
|
174
|
-
allowSubdomains?: boolean;
|
|
175
|
-
/** Strip repeated site chrome from per-page output. Defaults to true. */
|
|
176
|
-
stripBoilerplate?: boolean;
|
|
177
|
-
/** Fraction of pages a block must repeat in to count as chrome (0..1). Defaults to 0.5. */
|
|
178
|
-
boilerplateThreshold?: number;
|
|
179
|
-
verbose?: boolean;
|
|
180
|
-
/** Suppress all diagnostic/progress logging (issue #100). */
|
|
181
|
-
silent?: boolean;
|
|
182
|
-
artifacts?: ('llms.txt' | 'llms-full.txt' | 'markdown')[];
|
|
183
|
-
hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
|
|
178
|
+
/**
|
|
179
|
+
* Parsed URL patterns that replace the patterns parsed from `urls`.
|
|
180
|
+
* Advanced: `urls` covers most needs.
|
|
181
|
+
*/
|
|
182
|
+
globPatterns?: ParsedUrlPattern[];
|
|
184
183
|
}
|
|
185
|
-
|
|
184
|
+
/** Type helper for `mdream.config.*`. Every option is optional in the config file. */
|
|
185
|
+
export declare function defineConfig(config: Partial<CrawlOptions>): Partial<CrawlOptions>;
|
|
186
186
|
interface ParsedUrlPattern {
|
|
187
187
|
baseUrl: string;
|
|
188
188
|
pattern: string;
|
|
@@ -231,5 +231,10 @@ interface CrawlProgress {
|
|
|
231
231
|
current?: string;
|
|
232
232
|
};
|
|
233
233
|
}
|
|
234
|
+
/**
|
|
235
|
+
* Crawl a site and write the requested artifacts. Options are parsed once with
|
|
236
|
+
* `resolveCrawlOptions`; an unknown key or an invalid value throws a `TypeError`
|
|
237
|
+
* before any request is sent.
|
|
238
|
+
*/
|
|
234
239
|
export declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
|
|
235
|
-
export type { BoilerplateOptions, CrawlHooks, CrawlLogger, CrawlOptions, CrawlResult, CrawlSpinner,
|
|
240
|
+
export type { BoilerplateOptions, CrawlArtifact, CrawlDriver, CrawlHooks, CrawlLogger, CrawlOptions, CrawlProgress, CrawlResult, CrawlSpinner, PageData, PageMetadata, ParsedUrlPattern };
|
package/package.json
CHANGED
|
@@ -1,13 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/crawl",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "2.0.0",
|
|
5
5
|
"description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
8
8
|
"email": "harlan@harlanzw.com",
|
|
9
9
|
"url": "https://harlanzw.com/"
|
|
10
10
|
},
|
|
11
|
+
"contributors": [
|
|
12
|
+
{
|
|
13
|
+
"name": "André Cruz",
|
|
14
|
+
"url": "https://github.com/edevil"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"name": "Bernardo Pereira",
|
|
18
|
+
"url": "https://github.com/BernardoPe"
|
|
19
|
+
}
|
|
20
|
+
],
|
|
11
21
|
"license": "MIT",
|
|
12
22
|
"repository": {
|
|
13
23
|
"type": "git",
|
|
@@ -35,7 +45,7 @@
|
|
|
35
45
|
"main": "./dist/index.mjs",
|
|
36
46
|
"types": "./dist/index.d.mts",
|
|
37
47
|
"bin": {
|
|
38
|
-
"
|
|
48
|
+
"mdream-crawl": "./bin/mdream-crawl.mjs"
|
|
39
49
|
},
|
|
40
50
|
"files": [
|
|
41
51
|
"bin",
|
|
@@ -54,16 +64,16 @@
|
|
|
54
64
|
}
|
|
55
65
|
},
|
|
56
66
|
"dependencies": {
|
|
57
|
-
"@clack/prompts": "^1.
|
|
58
|
-
"@mdream/js": "
|
|
59
|
-
"c12": "4.0.0-rc.
|
|
60
|
-
"hookable": "^6.1.
|
|
61
|
-
"mdream": "
|
|
62
|
-
"nypm": "^0.6.
|
|
67
|
+
"@clack/prompts": "^1.8.1",
|
|
68
|
+
"@mdream/js": "2.0.0",
|
|
69
|
+
"c12": "4.0.0-rc.2",
|
|
70
|
+
"hookable": "^6.1.2",
|
|
71
|
+
"mdream": "2.0.0",
|
|
72
|
+
"nypm": "^0.6.10",
|
|
63
73
|
"ofetch": "^1.5.1",
|
|
64
74
|
"pathe": "^2.0.3",
|
|
65
75
|
"picomatch": "^4.0.7",
|
|
66
|
-
"tldts": "^7.4.
|
|
76
|
+
"tldts": "^7.4.16",
|
|
67
77
|
"ufo": "^1.6.4"
|
|
68
78
|
},
|
|
69
79
|
"devDependencies": {
|