@mdream/js 2.0.0-beta.1 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +402 -126
- package/dist/_chunks/clean.mjs +526 -0
- package/dist/_chunks/const.mjs +3 -7
- package/dist/_chunks/extraction.d.mts +14 -0
- package/dist/_chunks/html.mjs +260 -0
- package/dist/_chunks/markdown-processor.mjs +1077 -828
- package/dist/_chunks/minimal.mjs +29 -0
- package/dist/_chunks/output-runner.mjs +54 -0
- package/dist/_chunks/parse.mjs +134 -1501
- package/dist/_chunks/plugin.mjs +14 -0
- package/dist/_chunks/src.mjs +291 -0
- package/dist/_chunks/tag-overrides.mjs +138 -0
- package/dist/_chunks/{plugins.mjs → tailwind.mjs} +145 -292
- package/dist/_chunks/text.mjs +583 -0
- package/dist/_chunks/types.d.mts +136 -126
- package/dist/_chunks/url.mjs +82 -0
- package/dist/_chunks/utils.mjs +659 -0
- package/dist/clean.d.mts +10 -0
- package/dist/clean.mjs +2 -0
- package/dist/cli.mjs +47 -8
- package/dist/html.d.mts +6 -0
- package/dist/html.mjs +2 -0
- package/dist/index.d.mts +5 -4
- package/dist/index.mjs +4 -58
- package/dist/parse.d.mts +15 -10
- package/dist/plugins.d.mts +13 -28
- package/dist/plugins.mjs +52 -2
- package/dist/preset/minimal.d.mts +14 -2
- package/dist/preset/minimal.mjs +1 -32
- package/dist/splitter.d.mts +12 -4
- package/dist/splitter.mjs +386 -113
- package/dist/text.d.mts +6 -0
- package/dist/text.mjs +2 -0
- package/package.json +34 -8
- package/dist/_chunks/index.d.mts +0 -9
- package/dist/_chunks/minimal.d.mts +0 -6
- package/dist/_chunks/resolve-plugins.mjs +0 -44
- package/dist/_chunks/stream.mjs +0 -37
- package/dist/core.d.mts +0 -15
- package/dist/core.mjs +0 -28
package/README.md
CHANGED
|
@@ -6,35 +6,263 @@
|
|
|
6
6
|
|
|
7
7
|
## Installation
|
|
8
8
|
|
|
9
|
+
These examples use v2.
|
|
10
|
+
For `2.0.0-beta.1`, read the version note in [Migrating from v1](#migrating-from-v1) first.
|
|
11
|
+
|
|
9
12
|
```bash
|
|
10
13
|
# pnpm
|
|
11
|
-
pnpm add @mdream/js
|
|
14
|
+
pnpm add @mdream/js
|
|
12
15
|
|
|
13
16
|
# npm
|
|
14
|
-
npm install @mdream/js
|
|
17
|
+
npm install @mdream/js
|
|
15
18
|
|
|
16
19
|
# yarn
|
|
17
|
-
yarn add @mdream/js
|
|
20
|
+
yarn add @mdream/js
|
|
18
21
|
```
|
|
19
22
|
|
|
20
23
|
## Entry Points
|
|
21
24
|
|
|
22
25
|
| Import | Description |
|
|
23
26
|
|---|---|
|
|
24
|
-
| `@mdream/js` |
|
|
25
|
-
| `@mdream/js/
|
|
26
|
-
| `@mdream/js/
|
|
27
|
-
| `@mdream/js/
|
|
27
|
+
| `@mdream/js` | Tree-shakable Markdown conversion and streaming |
|
|
28
|
+
| `@mdream/js/text` | Tree-shakable plain text conversion and streaming |
|
|
29
|
+
| `@mdream/js/html` | Tree-shakable safe HTML conversion and streaming |
|
|
30
|
+
| `@mdream/js/clean` | `clean()` cleanup rules for the `clean` option |
|
|
31
|
+
| `@mdream/js/plugins` | Optional plugin factories |
|
|
32
|
+
| `@mdream/js/preset/minimal` | Explicit composition of the minimal plugin set |
|
|
28
33
|
| `@mdream/js/negotiate` | HTTP content negotiation: `shouldServeMarkdown`, `parseAcceptHeader` |
|
|
29
34
|
| `@mdream/js/parse` | Low-level HTML parser: `parseHtml`, `parseHtmlStream` |
|
|
30
35
|
| `@mdream/js/splitter` | Single-pass markdown splitter: `htmlToMarkdownSplitChunks`, `htmlToMarkdownSplitChunksStream` |
|
|
31
36
|
| `@mdream/js/llms-txt` | llms.txt artifact generation: `generateLlmsTxtArtifacts`, `createLlmsTxtStream` |
|
|
32
37
|
|
|
38
|
+
## Migrating from v1
|
|
39
|
+
|
|
40
|
+
Using a coding agent? Give it the [v1 to v2 migration prompt](https://github.com/harlan-zw/mdream/blob/main/MIGRATION_PROMPT.md).
|
|
41
|
+
|
|
42
|
+
This guide covers v2.
|
|
43
|
+
`@mdream/js@2.0.0-beta.1` uses the declarative plugin object, `hooks`, and `format` option.
|
|
44
|
+
For that release's examples, use the [beta.1 README](https://github.com/harlan-zw/mdream/blob/v2.0.0-beta.1/packages/js/README.md).
|
|
45
|
+
|
|
46
|
+
In v2, each output format and each plugin is a separate import.
|
|
47
|
+
Your bundle contains only the code that you import.
|
|
48
|
+
This guide applies to `@mdream/js` only.
|
|
49
|
+
For the Rust engine, see the [`mdream` migration guide](../mdream/README.md#migrating-from-v1).
|
|
50
|
+
Each converter throws a `TypeError` for an option that it does not read, such as a typo.
|
|
51
|
+
In v1, it ignored these options.
|
|
52
|
+
For a removed option, such as `format`, `hooks`, or a `plugins` object, the error names the fix.
|
|
53
|
+
The top-level `mdream` options `minimal`, `frontmatter`, `isolateMain`, `tailwind`, `filter`, and `extraction` also throw.
|
|
54
|
+
`cleanUrls` throws too. Pass `clean: clean({ urls: true })` with `clean` from `@mdream/js/clean`.
|
|
55
|
+
An option set to `undefined` does not throw.
|
|
56
|
+
The splitter also reads its chunk options, such as `chunkSize`.
|
|
57
|
+
The `blankLines` cleanup rule is removed. It had no effect, so delete it.
|
|
58
|
+
|
|
59
|
+
### Output formats
|
|
60
|
+
|
|
61
|
+
The `format` option is removed. Import the converter for the format.
|
|
62
|
+
|
|
63
|
+
| v1 | v2 |
|
|
64
|
+
|---|---|
|
|
65
|
+
| `htmlToMarkdown(html, { format: 'text' })` | `htmlToText(html)` from `@mdream/js/text` |
|
|
66
|
+
| `htmlToMarkdown(html, { format: 'html' })` | `htmlToSafeHtml(html)` from `@mdream/js/html` |
|
|
67
|
+
| `streamHtmlToMarkdown(stream, { format: 'text' })` | `streamHtmlToText(stream)` from `@mdream/js/text` |
|
|
68
|
+
| `streamHtmlToMarkdown(stream, { format: 'html' })` | `streamHtmlToSafeHtml(stream)` from `@mdream/js/html` |
|
|
69
|
+
|
|
70
|
+
The text and HTML converters accept `origin`, `plugins`, and `tagOverrides`.
|
|
71
|
+
`htmlToText` also accepts `wrapWidth`.
|
|
72
|
+
Both apply the `urls` rule from `clean`. `htmlToText` also applies `emptyImages`.
|
|
73
|
+
They skip the Markdown post-processing pass, as in v1.
|
|
74
|
+
The CLI `--format` flag is unchanged.
|
|
75
|
+
`frontmatterPlugin` now writes YAML frontmatter to Markdown output only.
|
|
76
|
+
In v1, safe HTML output started with a YAML block. This also applies to `--preset minimal --format html`.
|
|
77
|
+
To read the metadata in any format, use `onExtract`.
|
|
78
|
+
|
|
79
|
+
```diff
|
|
80
|
+
- import { htmlToMarkdown } from '@mdream/js'
|
|
81
|
+
- const text = htmlToMarkdown(html, { format: 'text' })
|
|
82
|
+
+ import { htmlToText } from '@mdream/js/text'
|
|
83
|
+
+ const text = htmlToText(html)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### Plugins
|
|
87
|
+
|
|
88
|
+
The declarative `plugins` object and the `hooks` option are removed.
|
|
89
|
+
`plugins` is now an array of plugin instances. Import each factory from `@mdream/js/plugins`.
|
|
90
|
+
|
|
91
|
+
| v1 | v2 |
|
|
92
|
+
|---|---|
|
|
93
|
+
| `plugins: { frontmatter: true }` | `plugins: [frontmatterPlugin()]` |
|
|
94
|
+
| `plugins: { frontmatter: fm => {} }` | `plugins: [frontmatterPlugin({ onExtract: fm => {} })]` |
|
|
95
|
+
| `plugins: { frontmatter: { additionalFields, metaFields, onExtract } }` | `plugins: [frontmatterPlugin({ additionalFields, metaFields, onExtract })]` |
|
|
96
|
+
| `plugins: { isolateMain: true }` | `plugins: [isolateMainPlugin()]` |
|
|
97
|
+
| `plugins: { tailwind: true }` | `plugins: [tailwindPlugin()]` |
|
|
98
|
+
| `plugins: { filter: { exclude, include, processChildren } }` | `plugins: [filterPlugin({ exclude, include, processChildren })]` |
|
|
99
|
+
| `plugins: { extraction: { selector: fn } }` | `plugins: [extractionPlugin({ selector: fn })]` (see below) |
|
|
100
|
+
| `plugins: { tagOverrides }` | top-level `tagOverrides` |
|
|
101
|
+
| `hooks: [myPlugin]` | `plugins: [myPlugin]` |
|
|
102
|
+
|
|
103
|
+
```diff
|
|
104
|
+
import { htmlToMarkdown } from '@mdream/js'
|
|
105
|
+
+ import { filterPlugin, frontmatterPlugin } from '@mdream/js/plugins'
|
|
106
|
+
|
|
107
|
+
htmlToMarkdown(html, {
|
|
108
|
+
- plugins: {
|
|
109
|
+
- frontmatter: true,
|
|
110
|
+
- filter: { exclude: ['nav'] },
|
|
111
|
+
- tagOverrides: { 'x-heading': 'h2' },
|
|
112
|
+
- },
|
|
113
|
+
- hooks: [myPlugin],
|
|
114
|
+
+ plugins: [frontmatterPlugin(), filterPlugin({ exclude: ['nav'] }), myPlugin],
|
|
115
|
+
+ tagOverrides: { 'x-heading': 'h2' },
|
|
116
|
+
})
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Plugins run in array order.
|
|
120
|
+
In v1, the built-in plugins always ran first, in this order: frontmatter, isolateMain, tailwind, filter, extraction.
|
|
121
|
+
`hooks` ran after them. To keep the v1 result, use the same order.
|
|
122
|
+
|
|
123
|
+
### Callback changes
|
|
124
|
+
|
|
125
|
+
- `frontmatterPlugin({ onExtract })` runs once after the conversion finishes, as in v1. It now also runs for streams, when the stream ends.
|
|
126
|
+
- `onExtract` now removes the YAML escape from backslashes. In v1, the title `a \ b` came back as `a \\ b`.
|
|
127
|
+
- `extractionPlugin` callbacks run when each matching element closes. In v1, the declarative `extraction` callbacks ran after the conversion finished.
|
|
128
|
+
- `extractionPlugin` callbacks receive the parsed element and the `PluginState`. The element does not have `selector` or `tagName`. Use `element.name` for the tag name. Use the object key or a separate callback to identify the selector.
|
|
129
|
+
|
|
130
|
+
```diff
|
|
131
|
+
- plugins: {
|
|
132
|
+
- extraction: {
|
|
133
|
+
- 'img[alt]': el => images.push({ tag: el.tagName, src: el.attributes.src }),
|
|
134
|
+
- },
|
|
135
|
+
- },
|
|
136
|
+
+ plugins: [
|
|
137
|
+
+ extractionPlugin({
|
|
138
|
+
+ 'img[alt]': el => images.push({ tag: el.name, src: el.attributes.src }),
|
|
139
|
+
+ }),
|
|
140
|
+
+ ],
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
### Splitter headings, filter selectors, and hook state
|
|
144
|
+
|
|
145
|
+
| v1 | v2 |
|
|
146
|
+
|---|---|
|
|
147
|
+
| `import { TAG_H2, TAG_H3 } from '@mdream/js'` | Remove the import. The root export has no `TAG_H1` to `TAG_H6`. |
|
|
148
|
+
| `headersToSplitOn: [TAG_H2, TAG_H3]` | `headersToSplitOn: [2, 3]` |
|
|
149
|
+
| A tag id number in `filterPlugin` `include` or `exclude` | The tag name, such as `filterPlugin({ exclude: ['form'] })` |
|
|
150
|
+
| Hook `state` with the converter's internal fields | Hook `state` typed as `PluginState` |
|
|
151
|
+
|
|
152
|
+
`headersToSplitOn` takes heading levels from 1 to 6.
|
|
153
|
+
If it gets another value, the splitter throws a `TypeError`. A stale `TAG_H2` (8) fails this way.
|
|
154
|
+
`filterPlugin` takes only strings in `include` and `exclude`.
|
|
155
|
+
A tag name compiles to the same fast tag match that a tag id gave.
|
|
156
|
+
If an entry is not a string, `filterPlugin` throws a `TypeError`.
|
|
157
|
+
|
|
158
|
+
Each hook gets a read-only [`PluginState`](#pluginstate) with `options`, `outputFormat`, and `depth`.
|
|
159
|
+
The hook type no longer shows the converter's internal fields, such as `buffer` and `depthMap`.
|
|
160
|
+
To keep data for a node, use `node.context`. To keep data for one document, use a [setup function](#plugins-with-per-document-state).
|
|
161
|
+
|
|
162
|
+
`ParseState` from `@mdream/js/parse` no longer has the quote fields that the parser stopped using.
|
|
163
|
+
These are `inSingleQuote`, `inDoubleQuote`, `inBacktick`, `inRawTextQuoteAware`, and `lastCharWasBackslash`.
|
|
164
|
+
`PluginContext` no longer has the fields of the removed readability plugin.
|
|
165
|
+
|
|
166
|
+
### Splitter output and timing
|
|
167
|
+
|
|
168
|
+
The splitter changes from [#316](https://github.com/harlan-zw/mdream/pull/316) also affect earlier v2 betas.
|
|
169
|
+
Old chunk IDs and line ranges may identify different text after this upgrade.
|
|
170
|
+
|
|
171
|
+
1. Rebuild stored chunks and embeddings from the original HTML.
|
|
172
|
+
2. Replace the stored chunk text and `metadata.loc` together.
|
|
173
|
+
3. Switch your application to the rebuilt index.
|
|
174
|
+
|
|
175
|
+
- Overlap starts at a word boundary. Chunk text and boundaries can change.
|
|
176
|
+
- A fenced code block stays in one chunk, even if it exceeds `chunkSize`.
|
|
177
|
+
- Chunks omit separators and leading or trailing blank lines, including with `clean()` or `withMinimalPreset()`.
|
|
178
|
+
- With `stripHeaders: true`, `metadata.loc` covers the kept content lines, excluding the removed heading.
|
|
179
|
+
- `htmlToMarkdownSplitChunksStream` converts the whole document before yielding its first chunk.
|
|
180
|
+
|
|
181
|
+
The generator keeps the converted Markdown in memory.
|
|
182
|
+
Stopping it early skips the remaining chunk work. HTML conversion has already finished.
|
|
183
|
+
If you need output during conversion, use `streamHtmlToMarkdown`.
|
|
184
|
+
It yields Markdown pieces without chunk metadata.
|
|
185
|
+
The splitter accepts `keepSeparator`, but drops whitespace between chunks.
|
|
186
|
+
|
|
187
|
+
### Cleanup
|
|
188
|
+
|
|
189
|
+
The `clean` option now takes the result of `clean()` from `@mdream/js/clean`.
|
|
190
|
+
The cleanup code is only in your bundle when you import it.
|
|
191
|
+
|
|
192
|
+
| v1 | v2 |
|
|
193
|
+
|---|---|
|
|
194
|
+
| `clean: true` | `clean: clean()` |
|
|
195
|
+
| `clean: { urls: true, fragments: true }` | `clean: clean({ urls: true, fragments: true })` |
|
|
196
|
+
|
|
197
|
+
```diff
|
|
198
|
+
import { htmlToMarkdown } from '@mdream/js'
|
|
199
|
+
+ import { clean } from '@mdream/js/clean'
|
|
200
|
+
|
|
201
|
+
- htmlToMarkdown(html, { clean: true })
|
|
202
|
+
+ htmlToMarkdown(html, { clean: clean() })
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
The rules now act on the links and images that the converter writes, as in the Rust engine.
|
|
206
|
+
Escaped brackets and code keep their text, and `streamHtmlToMarkdown` applies the rules too.
|
|
207
|
+
With `fragments`, a stream yields all output when the document ends, because a link can point to a later heading.
|
|
208
|
+
|
|
209
|
+
If a converter gets `clean: true` or a plain rules object, it throws a `TypeError`.
|
|
210
|
+
`withMinimalPreset` still enables all cleanup by default. To turn it off, pass `clean: false`.
|
|
211
|
+
|
|
212
|
+
### Plugin instances
|
|
213
|
+
|
|
214
|
+
You can create plugins once and reuse them for many conversions.
|
|
215
|
+
Each conversion gets fresh state from the built-in plugins.
|
|
216
|
+
If your own plugin keeps state for one document, use the setup form of `createPlugin`. See [Plugins with per-document state](#plugins-with-per-document-state).
|
|
217
|
+
|
|
218
|
+
### Minimal preset
|
|
219
|
+
|
|
220
|
+
- Import `withMinimalPreset` from `@mdream/js/preset/minimal`. The root export is removed.
|
|
221
|
+
- The preset returns plugin instances. Plugins that you pass are added after the preset plugins.
|
|
222
|
+
- An appended `filterPlugin()` adds exclusions. It cannot restore elements excluded by the preset.
|
|
223
|
+
- You cannot disable or reconfigure one preset plugin through the preset. To do that, compose the plugins yourself:
|
|
224
|
+
|
|
225
|
+
```ts
|
|
226
|
+
import { htmlToMarkdown } from '@mdream/js'
|
|
227
|
+
import { clean } from '@mdream/js/clean'
|
|
228
|
+
import { filterPlugin, isolateMainPlugin, tailwindPlugin } from '@mdream/js/plugins'
|
|
229
|
+
|
|
230
|
+
// Minimal preset without frontmatter, with a custom filter
|
|
231
|
+
htmlToMarkdown(html, {
|
|
232
|
+
clean: clean(),
|
|
233
|
+
plugins: [isolateMainPlugin(), tailwindPlugin(), filterPlugin({ exclude: ['nav', 'footer'] })],
|
|
234
|
+
})
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
### Output changes
|
|
238
|
+
|
|
239
|
+
`<title>` text no longer appears in the body, in any format.
|
|
240
|
+
Use `frontmatterPlugin()` for Markdown metadata, or its `onExtract` callback to read the title separately.
|
|
241
|
+
|
|
242
|
+
Streams now join to the same output as one-shot conversion.
|
|
243
|
+
Chunk boundaries can change. Join chunks before comparing a complete document.
|
|
244
|
+
The parser also preserves emoji split between chunks and decodes byte chunks across UTF-8 boundaries.
|
|
245
|
+
Review saved output for parser, whitespace, plugin, and cleanup fixes from
|
|
246
|
+
[#242](https://github.com/harlan-zw/mdream/pull/242), [#243](https://github.com/harlan-zw/mdream/pull/243),
|
|
247
|
+
and [#241](https://github.com/harlan-zw/mdream/pull/241).
|
|
248
|
+
The converter also removes trailing spaces from link text before empty elements. See [#318](https://github.com/harlan-zw/mdream/pull/318).
|
|
249
|
+
Regenerate snapshots from reviewed output; do not depend on individual stream boundaries.
|
|
250
|
+
|
|
251
|
+
### Removed exports
|
|
252
|
+
|
|
253
|
+
| v1 | v2 |
|
|
254
|
+
|---|---|
|
|
255
|
+
| `@mdream/js/core` | `@mdream/js`. The root entry is now tree-shakable. |
|
|
256
|
+
| `withMinimalPreset` from `@mdream/js` | `withMinimalPreset` from `@mdream/js/preset/minimal` |
|
|
257
|
+
| `extractionCollectorPlugin` | `extractionPlugin` |
|
|
258
|
+
| `BuiltinPlugins` type | `Plugin[]` |
|
|
259
|
+
| `FrontmatterConfig` type | `FrontmatterPluginOptions` from `@mdream/js/plugins` |
|
|
260
|
+
| `CoreOptions` type | `MdreamOptions` |
|
|
261
|
+
|
|
33
262
|
## API Reference
|
|
34
263
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
plugins.
|
|
264
|
+
Each output format has its own entry point. Import only the format and plugins
|
|
265
|
+
that your application uses.
|
|
38
266
|
|
|
39
267
|
### `htmlToMarkdown(html, options?)`
|
|
40
268
|
|
|
@@ -45,15 +273,27 @@ import { htmlToMarkdown } from '@mdream/js'
|
|
|
45
273
|
|
|
46
274
|
const md = htmlToMarkdown('<h1>Hello</h1><p>World</p>')
|
|
47
275
|
// # Hello\n\nWorld
|
|
276
|
+
```
|
|
48
277
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
278
|
+
### `htmlToText(html, options?)`
|
|
279
|
+
|
|
280
|
+
Converts HTML to readable plain text.
|
|
281
|
+
|
|
282
|
+
```typescript
|
|
283
|
+
import { htmlToText } from '@mdream/js/text'
|
|
284
|
+
|
|
285
|
+
const text = htmlToText('<h1>Hello</h1><p><strong>World</strong></p>')
|
|
52
286
|
// Hello\n\nWorld
|
|
287
|
+
```
|
|
53
288
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
289
|
+
### `htmlToSafeHtml(html, options?)`
|
|
290
|
+
|
|
291
|
+
Converts HTML to allowlisted semantic HTML.
|
|
292
|
+
|
|
293
|
+
```typescript
|
|
294
|
+
import { htmlToSafeHtml } from '@mdream/js/html'
|
|
295
|
+
|
|
296
|
+
const html = htmlToSafeHtml('<h1>Hello</h1><p><strong>World</strong></p>')
|
|
57
297
|
// <h1 id="hello">Hello</h1><p><strong>World</strong></p>
|
|
58
298
|
```
|
|
59
299
|
|
|
@@ -91,13 +331,14 @@ for await (const chunk of stream) {
|
|
|
91
331
|
|
|
92
332
|
**Returns:** `AsyncIterable<string>`
|
|
93
333
|
|
|
94
|
-
|
|
334
|
+
`streamHtmlToText` from `@mdream/js/text` and `streamHtmlToSafeHtml` from `@mdream/js/html` take the same parameters.
|
|
335
|
+
|
|
336
|
+
### Optional plugins
|
|
95
337
|
|
|
96
|
-
|
|
97
|
-
loading declarative built-in plugins or cleanup:
|
|
338
|
+
Import and compose each plugin explicitly.
|
|
98
339
|
|
|
99
340
|
```typescript
|
|
100
|
-
import { htmlToMarkdown } from '@mdream/js
|
|
341
|
+
import { htmlToMarkdown } from '@mdream/js'
|
|
101
342
|
import { filterPlugin } from '@mdream/js/plugins'
|
|
102
343
|
|
|
103
344
|
const markdown = htmlToMarkdown(html, {
|
|
@@ -114,32 +355,12 @@ const markdown = htmlToMarkdown(html, {
|
|
|
114
355
|
| Option | Type | Default | Description |
|
|
115
356
|
|---|---|---|---|
|
|
116
357
|
| `origin` | `string` | `undefined` | Origin URL for resolving relative image paths and internal links |
|
|
117
|
-
| `plugins` | `
|
|
118
|
-
| `
|
|
119
|
-
| `
|
|
358
|
+
| `plugins` | `Plugin[]` | `undefined` | Explicit conversion plugins, applied in array order |
|
|
359
|
+
| `tagOverrides` | `Record<string, TagOverride \| string>` | `undefined` | Custom tag output or aliases |
|
|
360
|
+
| `clean` | `Cleaner` | `undefined` | Cleanup rules from `clean()` in `@mdream/js/clean` (see [CleanOptions](#cleanoptions)). Streams apply them too. With `fragments`, a stream yields all output at the end. |
|
|
120
361
|
| `wrapWidth` | `number` | `undefined` | Hard-wrap prose at this many characters on word boundaries |
|
|
121
|
-
| `format` | `'markdown' \| 'text' \| 'html'` | `'markdown'` | Output Markdown, plain text, or HTML |
|
|
122
|
-
|
|
123
|
-
### `BuiltinPlugins`
|
|
124
|
-
|
|
125
|
-
Declarative configuration for built-in plugins. Works with both the JavaScript and Rust engines.
|
|
126
|
-
|
|
127
|
-
| Option | Type | Default | Description |
|
|
128
|
-
|---|---|---|---|
|
|
129
|
-
| `frontmatter` | `boolean \| ((fm: Record<string, string>) => void) \| FrontmatterConfig` | `undefined` | Extract metadata from HTML `<head>` into YAML frontmatter. Pass `true` for defaults, a callback to receive structured data, or a config object. |
|
|
130
|
-
| `isolateMain` | `boolean` | `undefined` | Isolate main content area. Prioritizes `<main>` elements, then falls back to header-to-footer heuristic. |
|
|
131
|
-
| `tailwind` | `boolean` | `undefined` | Convert Tailwind utility classes (bold, italic, hidden, etc.) to semantic Markdown formatting. |
|
|
132
|
-
| `filter` | `{ include?, exclude?, processChildren? }` | `undefined` | Filter elements by CSS selectors, tag names, or TAG_* constants (see [Filter Plugin](#filterpluginoptions)). |
|
|
133
|
-
| `extraction` | `Record<string, (element: ExtractedElement) => void>` | `undefined` | Extract elements matching CSS selectors during conversion. Each key is a CSS selector; the handler is called for every match. |
|
|
134
|
-
| `tagOverrides` | `Record<string, TagOverride \| string>` | `undefined` | Declarative tag overrides. String values act as aliases (e.g., `{ "x-heading": "h2" }`). Object values override specific handler properties. |
|
|
135
362
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
| Option | Type | Default | Description |
|
|
139
|
-
|---|---|---|---|
|
|
140
|
-
| `additionalFields` | `Record<string, string>` | `undefined` | Extra key-value pairs to inject into the frontmatter |
|
|
141
|
-
| `metaFields` | `string[]` | `['description', 'keywords', 'author', 'date', 'og:title', 'og:description', 'twitter:title', 'twitter:description']` | Meta tag names to extract beyond the standard set |
|
|
142
|
-
| `onExtract` | `(frontmatter: Record<string, string>) => void` | `undefined` | Callback to receive structured frontmatter data after conversion |
|
|
363
|
+
Any other key throws a `TypeError` that names the key.
|
|
143
364
|
|
|
144
365
|
### `TagOverride`
|
|
145
366
|
|
|
@@ -154,20 +375,27 @@ Declarative configuration for built-in plugins. Works with both the JavaScript a
|
|
|
154
375
|
|
|
155
376
|
### `CleanOptions`
|
|
156
377
|
|
|
157
|
-
|
|
378
|
+
Cleanup rules. Pass them to `clean()` from `@mdream/js/clean`:
|
|
379
|
+
|
|
380
|
+
```typescript
|
|
381
|
+
import { htmlToMarkdown } from '@mdream/js'
|
|
382
|
+
import { clean } from '@mdream/js/clean'
|
|
383
|
+
|
|
384
|
+
htmlToMarkdown(html, { clean: clean() }) // all rules
|
|
385
|
+
htmlToMarkdown(html, { clean: clean({ urls: true, fragments: true }) })
|
|
386
|
+
```
|
|
158
387
|
|
|
159
388
|
| Option | Type | Default | Description |
|
|
160
389
|
|---|---|---|---|
|
|
161
390
|
| `urls` | `boolean` | `false` | Strip tracking query parameters (`utm_*`, `fbclid`, `gclid`, etc.) from URLs |
|
|
162
391
|
| `fragments` | `boolean` | `false` | Strip fragment-only links that do not match any heading slug in the output |
|
|
163
392
|
| `emptyLinks` | `boolean` | `false` | Strip links with meaningless hrefs (`#`, `javascript:void(0)`, `data:`, `vbscript:`) and replace with plain text |
|
|
164
|
-
| `
|
|
165
|
-
| `
|
|
166
|
-
| `
|
|
167
|
-
| `emptyImages` | `boolean` | `false` | Strip images with no alt text (decorative images, tracking pixels) |
|
|
393
|
+
| `redundantLinks` | `boolean` | `false` | Strip links where text equals URL, with or without a title: `[https://x.com](https://x.com)` becomes `https://x.com` |
|
|
394
|
+
| `selfLinkHeadings` | `boolean` | `false` | Strip fragment links inside headings, such as permalink anchors: `## [Title](#title)` becomes `## Title` |
|
|
395
|
+
| `emptyImages` | `boolean` | `false` | Drop images with missing, empty, or whitespace-only alt text |
|
|
168
396
|
| `emptyLinkText` | `boolean` | `false` | Drop links that produce no visible text: `[](url)` is removed entirely |
|
|
169
397
|
|
|
170
|
-
|
|
398
|
+
`clean()` without rules enables all options.
|
|
171
399
|
|
|
172
400
|
---
|
|
173
401
|
|
|
@@ -176,6 +404,7 @@ When `clean: true` is passed, all options except `urls` and `blankLines` are ena
|
|
|
176
404
|
### `createPlugin(plugin)`
|
|
177
405
|
|
|
178
406
|
Factory function for creating type-safe transform plugins. All hooks are optional.
|
|
407
|
+
Each hook gets the read-only [`PluginState`](#pluginstate) of the conversion.
|
|
179
408
|
|
|
180
409
|
```typescript
|
|
181
410
|
import { createPlugin } from '@mdream/js/plugins'
|
|
@@ -204,23 +433,57 @@ const myPlugin = createPlugin({
|
|
|
204
433
|
|
|
205
434
|
| Hook | Parameters | Return Type | Description |
|
|
206
435
|
|---|---|---|---|
|
|
207
|
-
| `beforeNodeProcess` | `(event: NodeEvent, state)` | `{ skip: boolean } \| void` | Called before any node processing. Return `{ skip: true }` to skip the node. |
|
|
208
|
-
| `onNodeEnter` | `(element: ElementNode, state)` | `string \| void` | Called when entering an element. Return a string to inject markdown. |
|
|
209
|
-
| `onNodeExit` | `(element: ElementNode, state)` | `string \| void` | Called when exiting an element. Return a string to inject markdown. |
|
|
210
|
-
| `processAttributes` | `(element: ElementNode, state)` | `void` | Called to inspect or modify element attributes. |
|
|
211
|
-
| `processTextNode` | `(textNode: TextNode, state)` | `{ content: string, skip: boolean } \| void` | Called for each text node. Return an object to transform text or skip it. |
|
|
436
|
+
| `beforeNodeProcess` | `(event: NodeEvent, state: PluginState)` | `{ skip: boolean } \| void` | Called before any node processing. Return `{ skip: true }` to skip the node. |
|
|
437
|
+
| `onNodeEnter` | `(element: ElementNode, state: PluginState)` | `string \| void` | Called when entering an element. Return a string to inject markdown. |
|
|
438
|
+
| `onNodeExit` | `(element: ElementNode, state: PluginState)` | `string \| void` | Called when exiting an element. Return a string to inject markdown. |
|
|
439
|
+
| `processAttributes` | `(element: ElementNode, state: PluginState)` | `void` | Called to inspect or modify element attributes. |
|
|
440
|
+
| `processTextNode` | `(textNode: TextNode, state: PluginState)` | `{ content: string, skip: boolean } \| void` | Called for each text node. Return an object to transform text or skip it. |
|
|
441
|
+
| `onDocumentEnd` | `(state: PluginState)` | `void` | Called once after the whole document is converted, including for streams. |
|
|
442
|
+
|
|
443
|
+
#### `PluginState`
|
|
444
|
+
|
|
445
|
+
| Field | Type | Description |
|
|
446
|
+
|---|---|---|
|
|
447
|
+
| `options` | `Readonly<EngineOptions>` | The options passed to the converter |
|
|
448
|
+
| `outputFormat` | `'markdown' \| 'text' \| 'html'` | The output that the running converter writes |
|
|
449
|
+
| `depth` | `number` | The nesting depth of the current node. It is `0` before the first node. |
|
|
450
|
+
|
|
451
|
+
The state is read-only. To keep data for a node, set it on `node.context`.
|
|
452
|
+
|
|
453
|
+
#### Plugins with per-document state
|
|
454
|
+
|
|
455
|
+
If a plugin keeps state for one document, pass a setup function to `createPlugin`.
|
|
456
|
+
Each conversion calls it once and uses the hooks that it returns.
|
|
457
|
+
Then you can reuse one plugin array for many conversions, including concurrent streams.
|
|
458
|
+
|
|
459
|
+
```typescript
|
|
460
|
+
import { createPlugin } from '@mdream/js/plugins'
|
|
461
|
+
|
|
462
|
+
const headingCounter = createPlugin(() => {
|
|
463
|
+
let headings = 0
|
|
464
|
+
return {
|
|
465
|
+
onNodeEnter(element) {
|
|
466
|
+
if (element.name === 'h2')
|
|
467
|
+
headings++
|
|
468
|
+
},
|
|
469
|
+
}
|
|
470
|
+
})
|
|
471
|
+
```
|
|
472
|
+
|
|
473
|
+
The built-in `frontmatterPlugin`, `isolateMainPlugin`, and `extractionPlugin` use this form.
|
|
212
474
|
|
|
213
475
|
### `filterPlugin(options)`
|
|
214
476
|
|
|
215
|
-
Filters elements by CSS selectors
|
|
477
|
+
Filters elements by CSS selectors or tag names.
|
|
478
|
+
A tag name, such as `'nav'`, compiles to a fast tag match when you create the plugin.
|
|
479
|
+
If an entry is not a string, `filterPlugin` throws a `TypeError`.
|
|
216
480
|
|
|
217
481
|
```typescript
|
|
218
|
-
import { TAG_FOOTER, TAG_NAV } from '@mdream/js'
|
|
219
482
|
import { filterPlugin } from '@mdream/js/plugins'
|
|
220
483
|
|
|
221
484
|
// Exclude navigation and footer
|
|
222
485
|
const plugin = filterPlugin({
|
|
223
|
-
exclude: [
|
|
486
|
+
exclude: ['nav', 'footer', '.sidebar', '#ads'],
|
|
224
487
|
})
|
|
225
488
|
|
|
226
489
|
// Include only specific elements
|
|
@@ -232,8 +495,8 @@ const plugin2 = filterPlugin({
|
|
|
232
495
|
|
|
233
496
|
| Option | Type | Default | Description |
|
|
234
497
|
|---|---|---|---|
|
|
235
|
-
| `include` | `
|
|
236
|
-
| `exclude` | `
|
|
498
|
+
| `include` | `string[]` | `[]` | CSS selectors or tag names for elements to include (all others excluded) |
|
|
499
|
+
| `exclude` | `string[]` | `[]` | CSS selectors or tag names for elements to exclude |
|
|
237
500
|
| `processChildren` | `boolean` | `true` | Whether to also process children of matching elements |
|
|
238
501
|
|
|
239
502
|
### `frontmatterPlugin(options?)`
|
|
@@ -246,9 +509,18 @@ import { frontmatterPlugin } from '@mdream/js/plugins'
|
|
|
246
509
|
const plugin = frontmatterPlugin({
|
|
247
510
|
additionalFields: { source: 'crawler' },
|
|
248
511
|
metaFields: ['robots', 'viewport'],
|
|
512
|
+
onExtract: (frontmatter) => {
|
|
513
|
+
console.log(frontmatter.title)
|
|
514
|
+
},
|
|
249
515
|
})
|
|
250
516
|
```
|
|
251
517
|
|
|
518
|
+
| Option | Type | Description |
|
|
519
|
+
|---|---|---|
|
|
520
|
+
| `additionalFields` | `Record<string, string>` | Extra fields to add to the frontmatter |
|
|
521
|
+
| `metaFields` | `string[]` | Meta tag names to extract in addition to the defaults |
|
|
522
|
+
| `onExtract` | `(frontmatter: Record<string, string>) => void` | Receives the structured frontmatter once the conversion finishes. Runs for all formats and for streams. |
|
|
523
|
+
|
|
252
524
|
Default meta fields extracted: `description`, `keywords`, `author`, `date`, `og:title`, `og:description`, `twitter:title`, `twitter:description`.
|
|
253
525
|
|
|
254
526
|
### `isolateMainPlugin()`
|
|
@@ -285,11 +557,10 @@ const plugin = tailwindPlugin()
|
|
|
285
557
|
|
|
286
558
|
### `extractionPlugin(selectors)`
|
|
287
559
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
the
|
|
291
|
-
|
|
292
|
-
Extracts elements matching CSS selectors during conversion. Callbacks receive matching elements with their accumulated text content.
|
|
560
|
+
Extracts elements that match CSS selectors during conversion.
|
|
561
|
+
Each callback runs when a matching element closes.
|
|
562
|
+
It receives the parsed element with its trimmed `textContent`, and the [`PluginState`](#pluginstate).
|
|
563
|
+
Use `element.name` for the tag name and `element.attributes` for the attributes.
|
|
293
564
|
|
|
294
565
|
```typescript
|
|
295
566
|
import { extractionPlugin } from '@mdream/js/plugins'
|
|
@@ -304,17 +575,15 @@ const plugin = extractionPlugin({
|
|
|
304
575
|
})
|
|
305
576
|
```
|
|
306
577
|
|
|
307
|
-
### `extractionCollectorPlugin(selectors)`
|
|
308
|
-
|
|
309
|
-
Internal extraction collector for the `plugins.extraction` config. Collects results during processing and calls callbacks post-conversion to match Rust engine behavior.
|
|
310
|
-
|
|
311
578
|
---
|
|
312
579
|
|
|
313
580
|
## Presets
|
|
314
581
|
|
|
315
582
|
### `withMinimalPreset(options?)`
|
|
316
583
|
|
|
317
|
-
Returns
|
|
584
|
+
Returns frontmatter, isolate-main, Tailwind, and filter plugin instances.
|
|
585
|
+
It enables all cleanup rules by default.
|
|
586
|
+
To turn cleanup off, pass `clean: false`.
|
|
318
587
|
|
|
319
588
|
```typescript
|
|
320
589
|
import { htmlToMarkdown } from '@mdream/js'
|
|
@@ -327,17 +596,13 @@ const md = htmlToMarkdown(html, withMinimalPreset({
|
|
|
327
596
|
|
|
328
597
|
The minimal preset excludes these elements by default: `<form>`, `<fieldset>`, `<object>`, `<embed>`, `<footer>`, `<aside>`, `<iframe>`, `<input>`, `<textarea>`, `<select>`, `<button>`, `<nav>`.
|
|
329
598
|
|
|
330
|
-
You can
|
|
599
|
+
You can append custom plugins:
|
|
331
600
|
|
|
332
601
|
```typescript
|
|
333
602
|
const md = htmlToMarkdown(html, withMinimalPreset({
|
|
334
603
|
origin: 'https://example.com',
|
|
335
|
-
clean: { urls: true, fragments: true },
|
|
336
|
-
plugins:
|
|
337
|
-
frontmatter: {
|
|
338
|
-
additionalFields: { source: 'my-crawler' },
|
|
339
|
-
},
|
|
340
|
-
},
|
|
604
|
+
clean: clean({ urls: true, fragments: true }),
|
|
605
|
+
plugins: [myPlugin],
|
|
341
606
|
}))
|
|
342
607
|
```
|
|
343
608
|
|
|
@@ -354,13 +619,18 @@ Determines if a client prefers Markdown over HTML using HTTP content negotiation
|
|
|
354
619
|
- Bare wildcards (`*/*`) do not trigger Markdown (prevents breaking OG crawlers).
|
|
355
620
|
- `sec-fetch-dest: document` always returns `false` (browser navigation).
|
|
356
621
|
|
|
357
|
-
```
|
|
622
|
+
```ts
|
|
358
623
|
import { shouldServeMarkdown } from '@mdream/js/negotiate'
|
|
359
624
|
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
headers
|
|
363
|
-
|
|
625
|
+
export function markdownResponse(request: Request, markdown: string): Response | undefined {
|
|
626
|
+
if (shouldServeMarkdown(
|
|
627
|
+
request.headers.get('accept') ?? undefined,
|
|
628
|
+
request.headers.get('sec-fetch-dest') ?? undefined,
|
|
629
|
+
)) {
|
|
630
|
+
return new Response(markdown, {
|
|
631
|
+
headers: { 'Content-Type': 'text/markdown' },
|
|
632
|
+
})
|
|
633
|
+
}
|
|
364
634
|
}
|
|
365
635
|
```
|
|
366
636
|
|
|
@@ -415,7 +685,9 @@ for (const chunk of chunks) {
|
|
|
415
685
|
|
|
416
686
|
### `htmlToMarkdownSplitChunksStream(html, options?)`
|
|
417
687
|
|
|
418
|
-
|
|
688
|
+
Converts the whole document, then yields chunks one at a time.
|
|
689
|
+
It keeps the converted Markdown in memory, but avoids collecting the chunk array.
|
|
690
|
+
See [Splitter output and timing](#splitter-output-and-timing) before upgrading a stored index.
|
|
419
691
|
|
|
420
692
|
```typescript
|
|
421
693
|
import { htmlToMarkdownSplitChunksStream } from '@mdream/js/splitter'
|
|
@@ -433,13 +705,13 @@ Extends `EngineOptions` with chunking-specific settings.
|
|
|
433
705
|
|
|
434
706
|
| Option | Type | Default | Description |
|
|
435
707
|
|---|---|---|---|
|
|
436
|
-
| `headersToSplitOn` | `
|
|
708
|
+
| `headersToSplitOn` | `(1 \| 2 \| 3 \| 4 \| 5 \| 6)[]` | `[2, 3, 4, 5, 6]` | Heading levels that start a new chunk, from 1 (`<h1>`) to 6 (`<h6>`). Another value throws a `TypeError`. |
|
|
437
709
|
| `returnEachLine` | `boolean` | `false` | Return each non-empty line as an individual chunk |
|
|
438
710
|
| `stripHeaders` | `boolean` | `true` | Strip header lines from chunk content |
|
|
439
|
-
| `chunkSize` | `number` | `1000` |
|
|
711
|
+
| `chunkSize` | `number` | `1000` | Target chunk size, measured by `lengthFunction`. Complete code fences can exceed it. |
|
|
440
712
|
| `chunkOverlap` | `number` | `200` | Overlap between chunks for context preservation. Must be less than `chunkSize`. |
|
|
441
713
|
| `lengthFunction` | `(text: string) => number` | `(text) => text.length` | Function to measure chunk length. Replace with a token counter for LLM applications. |
|
|
442
|
-
| `keepSeparator` | `boolean` | `false` |
|
|
714
|
+
| `keepSeparator` | `boolean` | `false` | The splitter drops whitespace between chunks for either value. |
|
|
443
715
|
|
|
444
716
|
### `MarkdownChunk`
|
|
445
717
|
|
|
@@ -448,7 +720,7 @@ Extends `EngineOptions` with chunking-specific settings.
|
|
|
448
720
|
| `content` | `string` | The markdown content of the chunk |
|
|
449
721
|
| `metadata.headers` | `Record<string, string> \| undefined` | Header hierarchy at this chunk position (e.g., `{ h2: 'API', h3: 'Methods' }`) |
|
|
450
722
|
| `metadata.code` | `string \| undefined` | Code block language if chunk contains code |
|
|
451
|
-
| `metadata.loc` | `{ lines: { from: number, to: number } } \| undefined` |
|
|
723
|
+
| `metadata.loc` | `{ lines: { from: number, to: number } } \| undefined` | Inclusive, one-based line range in converted Markdown, excluding stripped headings. |
|
|
452
724
|
|
|
453
725
|
---
|
|
454
726
|
|
|
@@ -528,7 +800,9 @@ const result = await generateLlmsTxtArtifacts({
|
|
|
528
800
|
|
|
529
801
|
### `createLlmsTxtStream(options)`
|
|
530
802
|
|
|
531
|
-
Creates a `WritableStream<ProcessedFile>` that generates llms.txt artifacts
|
|
803
|
+
Creates a `WritableStream<ProcessedFile>` that generates llms.txt artifacts as pages arrive.
|
|
804
|
+
It writes full page content to disk and keeps page metadata in memory.
|
|
805
|
+
When the stream closes, it sorts the llms.txt listing by URL path hierarchy.
|
|
532
806
|
|
|
533
807
|
```typescript
|
|
534
808
|
import { createLlmsTxtStream } from '@mdream/js/llms-txt'
|
|
@@ -570,12 +844,19 @@ const { events, remainingHtml } = parseHtml('<p>Hello</p>')
|
|
|
570
844
|
Streaming parser that calls `onEvent` for each DOM event. Returns any remaining unparsed HTML (useful for processing partial chunks).
|
|
571
845
|
|
|
572
846
|
```typescript
|
|
573
|
-
import {
|
|
847
|
+
import type { NodeEvent } from '@mdream/js'
|
|
848
|
+
import { finalizeParse, parseHtmlStream } from '@mdream/js/parse'
|
|
574
849
|
|
|
575
|
-
const state = { depthMap: new
|
|
576
|
-
const
|
|
577
|
-
|
|
578
|
-
|
|
850
|
+
const state = { depthMap: new Uint16Array(1024), depth: 0 }
|
|
851
|
+
const onEvent = (event: NodeEvent) => console.log(event)
|
|
852
|
+
const chunks = ['<p>Hello ', 'world</p>']
|
|
853
|
+
let remaining = ''
|
|
854
|
+
|
|
855
|
+
for (const chunk of chunks) {
|
|
856
|
+
remaining = parseHtmlStream(remaining + chunk, state, onEvent)
|
|
857
|
+
}
|
|
858
|
+
// Flush trailing text and close open elements at the end of the input.
|
|
859
|
+
finalizeParse(remaining, state, onEvent)
|
|
579
860
|
```
|
|
580
861
|
|
|
581
862
|
**Returns:** `string` (remaining unparsed HTML)
|
|
@@ -615,6 +896,8 @@ curl -s https://example.com | npx @mdream/js --format html
|
|
|
615
896
|
| `-v, --version` | Show version number |
|
|
616
897
|
| `-h, --help` | Show help |
|
|
617
898
|
|
|
899
|
+
If you pass an unknown flag, a flag without a value, or an unknown preset, the CLI writes one line to stderr and exits with code 1.
|
|
900
|
+
|
|
618
901
|
---
|
|
619
902
|
|
|
620
903
|
## Usage Examples
|
|
@@ -624,45 +907,44 @@ curl -s https://example.com | npx @mdream/js --format html
|
|
|
624
907
|
Map custom HTML elements to standard Markdown behavior:
|
|
625
908
|
|
|
626
909
|
```typescript
|
|
627
|
-
import { htmlToMarkdown } from '@mdream/js
|
|
910
|
+
import { htmlToMarkdown } from '@mdream/js'
|
|
628
911
|
|
|
629
912
|
const md = htmlToMarkdown('<x-heading>Title</x-heading>', {
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
'
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
spacing: [2, 2],
|
|
640
|
-
},
|
|
913
|
+
tagOverrides: {
|
|
914
|
+
// String alias: make <x-heading> behave like <h2>
|
|
915
|
+
'x-heading': 'h2',
|
|
916
|
+
|
|
917
|
+
// Object override: custom enter/exit strings
|
|
918
|
+
'callout': {
|
|
919
|
+
enter: '> **Note:** ',
|
|
920
|
+
exit: '\n',
|
|
921
|
+
spacing: [2, 2],
|
|
641
922
|
},
|
|
642
923
|
},
|
|
643
924
|
})
|
|
644
925
|
```
|
|
645
926
|
|
|
646
|
-
###
|
|
927
|
+
### Extraction
|
|
647
928
|
|
|
648
929
|
Extract data from elements during conversion:
|
|
649
930
|
|
|
650
931
|
```typescript
|
|
651
932
|
import { htmlToMarkdown } from '@mdream/js'
|
|
933
|
+
import { extractionPlugin } from '@mdream/js/plugins'
|
|
652
934
|
|
|
653
935
|
const images: { src: string, alt: string }[] = []
|
|
654
936
|
|
|
655
937
|
const md = htmlToMarkdown(html, {
|
|
656
|
-
plugins:
|
|
657
|
-
|
|
938
|
+
plugins: [
|
|
939
|
+
extractionPlugin({
|
|
658
940
|
'img[alt]': (element) => {
|
|
659
941
|
images.push({
|
|
660
942
|
src: element.attributes.src,
|
|
661
943
|
alt: element.attributes.alt,
|
|
662
944
|
})
|
|
663
945
|
},
|
|
664
|
-
},
|
|
665
|
-
|
|
946
|
+
}),
|
|
947
|
+
],
|
|
666
948
|
})
|
|
667
949
|
|
|
668
950
|
console.log('Found images:', images)
|
|
@@ -674,19 +956,20 @@ Receive structured frontmatter data for further processing:
|
|
|
674
956
|
|
|
675
957
|
```typescript
|
|
676
958
|
import { htmlToMarkdown } from '@mdream/js'
|
|
959
|
+
import { frontmatterPlugin } from '@mdream/js/plugins'
|
|
677
960
|
|
|
678
961
|
let metadata: Record<string, string> = {}
|
|
679
962
|
|
|
680
963
|
const md = htmlToMarkdown(html, {
|
|
681
|
-
plugins:
|
|
682
|
-
|
|
964
|
+
plugins: [
|
|
965
|
+
frontmatterPlugin({
|
|
683
966
|
additionalFields: { source: 'crawler' },
|
|
684
967
|
metaFields: ['robots'],
|
|
685
968
|
onExtract: (fm) => {
|
|
686
969
|
metadata = fm
|
|
687
970
|
},
|
|
688
|
-
},
|
|
689
|
-
|
|
971
|
+
}),
|
|
972
|
+
],
|
|
690
973
|
})
|
|
691
974
|
|
|
692
975
|
console.log('Title:', metadata.title)
|
|
@@ -696,7 +979,7 @@ console.log('Description:', metadata.description)
|
|
|
696
979
|
### Composable Plugin for Content Filtering
|
|
697
980
|
|
|
698
981
|
```typescript
|
|
699
|
-
import { htmlToMarkdown } from '@mdream/js
|
|
982
|
+
import { htmlToMarkdown } from '@mdream/js'
|
|
700
983
|
import { createPlugin } from '@mdream/js/plugins'
|
|
701
984
|
|
|
702
985
|
const md = htmlToMarkdown(html, {
|
|
@@ -746,24 +1029,23 @@ const chunks = htmlToMarkdownSplitChunks(html, {
|
|
|
746
1029
|
|
|
747
1030
|
```typescript
|
|
748
1031
|
import type {
|
|
749
|
-
BuiltinPlugins,
|
|
750
1032
|
CleanOptions,
|
|
751
1033
|
ElementNode,
|
|
752
1034
|
EngineOptions,
|
|
753
1035
|
ExtractedElement,
|
|
754
|
-
FrontmatterConfig,
|
|
755
1036
|
MarkdownChunk,
|
|
756
1037
|
MdreamOptions,
|
|
757
1038
|
Node,
|
|
758
1039
|
NodeEvent,
|
|
1040
|
+
OutputFormat,
|
|
1041
|
+
Plugin,
|
|
759
1042
|
PluginContext,
|
|
1043
|
+
PluginState,
|
|
760
1044
|
SplitterOptions,
|
|
761
1045
|
TagOverride,
|
|
762
1046
|
TextNode,
|
|
763
1047
|
TransformPlugin,
|
|
764
1048
|
} from '@mdream/js'
|
|
765
|
-
|
|
766
|
-
import type { CoreOptions, Plugin } from '@mdream/js/core'
|
|
767
1049
|
```
|
|
768
1050
|
|
|
769
1051
|
## Exported Constants
|
|
@@ -773,18 +1055,12 @@ import {
|
|
|
773
1055
|
ELEMENT_NODE, // Node type for HTML elements (1)
|
|
774
1056
|
NodeEventEnter, // Event type for entering a node (0)
|
|
775
1057
|
NodeEventExit, // Event type for exiting a node (1)
|
|
776
|
-
TAG_H1, // Tag ID constant for <h1>
|
|
777
|
-
TAG_H2, // Tag ID constant for <h2>
|
|
778
|
-
TAG_H3, // Tag ID constant for <h3>
|
|
779
|
-
TAG_H4, // Tag ID constant for <h4>
|
|
780
|
-
TAG_H5, // Tag ID constant for <h5>
|
|
781
|
-
TAG_H6, // Tag ID constant for <h6>
|
|
782
1058
|
TEXT_NODE, // Node type for text content (3)
|
|
783
1059
|
} from '@mdream/js'
|
|
784
1060
|
```
|
|
785
1061
|
|
|
786
1062
|
|
|
787
|
-
[npm-version-src]: https://img.shields.io/npm/v/@mdream/js
|
|
1063
|
+
[npm-version-src]: https://img.shields.io/npm/v/@mdream/js.svg?style=flat&colorA=18181B&colorB=4C9BE0
|
|
788
1064
|
[npm-version-href]: https://npmjs.com/package/@mdream/js
|
|
789
1065
|
[npm-downloads-src]: https://img.shields.io/npm/dm/@mdream/js.svg?style=flat&colorA=18181B&colorB=4C9BE0
|
|
790
1066
|
[npm-downloads-href]: https://npm.chart.dev/@mdream/js
|