@mdream/crawl 1.1.2 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -1
- package/dist/_chunks/crawl.mjs +7 -0
- package/dist/index.d.mts +5 -0
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -301,7 +301,7 @@ CLI arguments override config file values. Array options like `exclude` are conc
|
|
|
301
301
|
|
|
302
302
|
## Hooks
|
|
303
303
|
|
|
304
|
-
|
|
304
|
+
Six hooks let you intercept and transform data at each stage of the crawl pipeline. Hooks receive mutable objects. Mutate in-place to transform output.
|
|
305
305
|
|
|
306
306
|
### `crawl:url`
|
|
307
307
|
|
|
@@ -319,6 +319,21 @@ defineConfig({
|
|
|
319
319
|
})
|
|
320
320
|
```
|
|
321
321
|
|
|
322
|
+
### `crawl:html`
|
|
323
|
+
|
|
324
|
+
Called after fetching, before HTML-to-Markdown conversion. Mutate `ctx.html` to transform the raw HTML (strip elements, inject content) before mdream processes it. Not called when the fetched content is already markdown.
|
|
325
|
+
|
|
326
|
+
```typescript
|
|
327
|
+
defineConfig({
|
|
328
|
+
hooks: {
|
|
329
|
+
'crawl:html': (ctx) => {
|
|
330
|
+
// ctx.url, ctx.html, ctx.origin
|
|
331
|
+
ctx.html = ctx.html.replace(/<header[\s\S]*?<\/header>/, '')
|
|
332
|
+
},
|
|
333
|
+
},
|
|
334
|
+
})
|
|
335
|
+
```
|
|
336
|
+
|
|
322
337
|
### `crawl:page`
|
|
323
338
|
|
|
324
339
|
Called after HTML-to-Markdown conversion, before storage. Mutate `page.title` or other fields. This replaces the `onPage` callback (which still works for backwards compatibility).
|
package/dist/_chunks/crawl.mjs
CHANGED
|
@@ -489,6 +489,13 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
489
489
|
links: []
|
|
490
490
|
};
|
|
491
491
|
} else {
|
|
492
|
+
const htmlCtx = {
|
|
493
|
+
url,
|
|
494
|
+
html: content,
|
|
495
|
+
origin: pageOrigin
|
|
496
|
+
};
|
|
497
|
+
await hooks.callHook("crawl:html", htmlCtx);
|
|
498
|
+
content = htmlCtx.html;
|
|
492
499
|
const { extraction, getMetadata } = extractMetadataInline(parsedUrl, allowedRegistrableDomains);
|
|
493
500
|
md = htmlToMarkdown(content, {
|
|
494
501
|
origin: pageOrigin,
|
package/dist/index.d.mts
CHANGED
|
@@ -10,6 +10,11 @@ interface CrawlHooks {
|
|
|
10
10
|
url: string;
|
|
11
11
|
skip: boolean;
|
|
12
12
|
}) => void | Promise<void>;
|
|
13
|
+
'crawl:html': (ctx: {
|
|
14
|
+
url: string;
|
|
15
|
+
html: string;
|
|
16
|
+
origin: string;
|
|
17
|
+
}) => void | Promise<void>;
|
|
13
18
|
'crawl:page': (page: PageData) => void | Promise<void>;
|
|
14
19
|
'crawl:content': (ctx: {
|
|
15
20
|
url: string;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/crawl",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.1
|
|
4
|
+
"version": "1.2.1",
|
|
5
5
|
"description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -63,8 +63,8 @@
|
|
|
63
63
|
"picomatch": "^4.0.4",
|
|
64
64
|
"tldts": "^7.0.30",
|
|
65
65
|
"ufo": "^1.6.4",
|
|
66
|
-
"@mdream/js": "1.1
|
|
67
|
-
"mdream": "1.1
|
|
66
|
+
"@mdream/js": "1.2.1",
|
|
67
|
+
"mdream": "1.2.1"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
70
|
"@types/picomatch": "^4.0.3"
|