@mdream/js 1.7.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +402 -126
- package/dist/_chunks/clean.mjs +526 -0
- package/dist/_chunks/const.mjs +3 -7
- package/dist/_chunks/extraction.d.mts +14 -0
- package/dist/_chunks/html.mjs +260 -0
- package/dist/_chunks/markdown-processor.mjs +1077 -827
- package/dist/_chunks/minimal.mjs +29 -0
- package/dist/_chunks/output-runner.mjs +54 -0
- package/dist/_chunks/parse.mjs +136 -1490
- package/dist/_chunks/plugin.mjs +14 -0
- package/dist/_chunks/src.mjs +291 -0
- package/dist/_chunks/tag-overrides.mjs +138 -0
- package/dist/_chunks/{plugins.mjs → tailwind.mjs} +145 -292
- package/dist/_chunks/text.mjs +583 -0
- package/dist/_chunks/types.d.mts +136 -122
- package/dist/_chunks/url.mjs +82 -0
- package/dist/_chunks/utils.mjs +659 -0
- package/dist/clean.d.mts +10 -0
- package/dist/clean.mjs +2 -0
- package/dist/cli.mjs +47 -8
- package/dist/html.d.mts +6 -0
- package/dist/html.mjs +2 -0
- package/dist/index.d.mts +5 -4
- package/dist/index.mjs +4 -34
- package/dist/parse.d.mts +15 -10
- package/dist/plugins.d.mts +13 -28
- package/dist/plugins.mjs +52 -2
- package/dist/preset/minimal.d.mts +14 -2
- package/dist/preset/minimal.mjs +1 -27
- package/dist/splitter.d.mts +12 -4
- package/dist/splitter.mjs +355 -113
- package/dist/text.d.mts +6 -0
- package/dist/text.mjs +2 -0
- package/package.json +33 -7
- package/dist/_chunks/index.d.mts +0 -9
- package/dist/_chunks/minimal.d.mts +0 -6
- package/dist/_chunks/resolve-plugins.mjs +0 -44
- package/dist/_chunks/stream.mjs +0 -37
- package/dist/core.d.mts +0 -15
- package/dist/core.mjs +0 -28
package/dist/_chunks/types.d.mts
CHANGED
|
@@ -1,18 +1,31 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
1
|
+
/** A boundary in writer-owned buffer fragments, before final cleanup. */
|
|
2
|
+
interface OutputPosition {
|
|
3
|
+
fragment: number;
|
|
4
|
+
offset: number;
|
|
5
|
+
kind: 'boundary' | 'content';
|
|
6
|
+
}
|
|
7
|
+
interface OutputPositions {
|
|
8
|
+
capture: (fragment?: number, offset?: number, kind?: OutputPosition['kind']) => OutputPosition;
|
|
9
|
+
replace: (start: number, end: number, replacement: string[], mapOffset?: (offset: number, kind: OutputPosition['kind']) => number) => void;
|
|
10
|
+
finish: () => (position: OutputPosition) => number;
|
|
11
|
+
}
|
|
7
12
|
export declare const ELEMENT_NODE = 1;
|
|
8
13
|
export declare const TEXT_NODE = 2;
|
|
9
14
|
declare const NodeEventEnter$1 = 0;
|
|
10
15
|
declare const NodeEventExit$1 = 1;
|
|
11
16
|
/**
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* For declarative config that works with both engines, use `BuiltinPlugins`.
|
|
17
|
+
* Conversion state that each plugin hook receives. The converter owns it:
|
|
18
|
+
* read it in a hook, and keep plugin data on `node.context` or in the plugin.
|
|
15
19
|
*/
|
|
20
|
+
export interface PluginState {
|
|
21
|
+
/** Options passed to the converter. */
|
|
22
|
+
readonly options: Readonly<EngineOptions>;
|
|
23
|
+
/** Output that the running converter writes. */
|
|
24
|
+
readonly outputFormat: OutputFormat;
|
|
25
|
+
/** Nesting depth of the current node. It is 0 before the first node. */
|
|
26
|
+
readonly depth: number;
|
|
27
|
+
}
|
|
28
|
+
/** Composable JavaScript conversion plugin. */
|
|
16
29
|
export interface TransformPlugin {
|
|
17
30
|
/**
|
|
18
31
|
* Whether a flattened element starts an output-excluded subtree.
|
|
@@ -22,31 +35,43 @@ export interface TransformPlugin {
|
|
|
22
35
|
/**
|
|
23
36
|
* Process a node before it's handled by the parser
|
|
24
37
|
*/
|
|
25
|
-
beforeNodeProcess?: (event: NodeEvent, state:
|
|
38
|
+
beforeNodeProcess?: (event: NodeEvent, state: PluginState) => undefined | void | {
|
|
26
39
|
skip: boolean;
|
|
27
40
|
};
|
|
28
41
|
/**
|
|
29
42
|
* Hook that runs when entering a node
|
|
30
43
|
* @returns String to add to the output, or PluginHookResult with content
|
|
31
44
|
*/
|
|
32
|
-
onNodeEnter?: (node: ElementNode, state:
|
|
45
|
+
onNodeEnter?: (node: ElementNode, state: PluginState) => string | undefined | void;
|
|
33
46
|
/**
|
|
34
47
|
* Hook that runs when exiting a node
|
|
35
48
|
*/
|
|
36
|
-
onNodeExit?: (node: ElementNode, state:
|
|
49
|
+
onNodeExit?: (node: ElementNode, state: PluginState) => string | undefined | void;
|
|
37
50
|
/**
|
|
38
51
|
* Process attributes for a node
|
|
39
52
|
*/
|
|
40
|
-
processAttributes?: (node: ElementNode, state:
|
|
53
|
+
processAttributes?: (node: ElementNode, state: PluginState) => void;
|
|
41
54
|
/**
|
|
42
55
|
* Process a text node before it's added to the output
|
|
43
56
|
* @returns Result with content and skip flag, or undefined for no transformation
|
|
44
57
|
*/
|
|
45
|
-
processTextNode?: (node: TextNode, state:
|
|
58
|
+
processTextNode?: (node: TextNode, state: PluginState) => {
|
|
46
59
|
content: string;
|
|
47
60
|
skip: boolean;
|
|
48
61
|
} | undefined;
|
|
62
|
+
/**
|
|
63
|
+
* Runs once after the whole document is converted, including for streams.
|
|
64
|
+
*/
|
|
65
|
+
onDocumentEnd?: (state: PluginState) => void;
|
|
49
66
|
}
|
|
67
|
+
/**
|
|
68
|
+
* Creates fresh hooks for one conversion. Use it for a plugin that keeps
|
|
69
|
+
* per-document state, so a reused or concurrent plugin never shares that state.
|
|
70
|
+
*/
|
|
71
|
+
export type PluginSetup = () => TransformPlugin;
|
|
72
|
+
/** Hooks shared by every conversion, or a setup that creates them per conversion. */
|
|
73
|
+
export type Plugin = TransformPlugin | PluginSetup;
|
|
74
|
+
export type OutputFormat = 'markdown' | 'text' | 'html';
|
|
50
75
|
/**
|
|
51
76
|
* Declarative tag override configuration.
|
|
52
77
|
* When a string value is provided, it acts as an alias (e.g. `{ "x-heading": "h2" }`).
|
|
@@ -59,56 +84,7 @@ export interface TagOverride {
|
|
|
59
84
|
isSelfClosing?: boolean;
|
|
60
85
|
collapsesInnerWhiteSpace?: boolean;
|
|
61
86
|
}
|
|
62
|
-
/**
|
|
63
|
-
* Frontmatter configuration options.
|
|
64
|
-
*/
|
|
65
|
-
export interface FrontmatterConfig {
|
|
66
|
-
additionalFields?: Record<string, string>;
|
|
67
|
-
metaFields?: string[];
|
|
68
|
-
/**
|
|
69
|
-
* Callback to receive structured frontmatter data.
|
|
70
|
-
* Called after conversion with the extracted key-value pairs.
|
|
71
|
-
*/
|
|
72
|
-
onExtract?: (frontmatter: Record<string, string>) => void;
|
|
73
|
-
}
|
|
74
|
-
/**
|
|
75
|
-
* Declarative configuration for built-in plugins.
|
|
76
|
-
* Works with both the JavaScript and Rust engines.
|
|
77
|
-
*/
|
|
78
|
-
export interface BuiltinPlugins {
|
|
79
|
-
/** Filter elements by CSS selectors, tag names, or TAG_* constants */
|
|
80
|
-
filter?: {
|
|
81
|
-
include?: (string | number)[];
|
|
82
|
-
exclude?: (string | number)[];
|
|
83
|
-
processChildren?: boolean;
|
|
84
|
-
};
|
|
85
|
-
/**
|
|
86
|
-
* Extract frontmatter from HTML head.
|
|
87
|
-
* - `true`: enable with defaults
|
|
88
|
-
* - `(fm) => void`: enable and receive structured data via callback
|
|
89
|
-
* - `FrontmatterConfig`: enable with config options and optional callback
|
|
90
|
-
*/
|
|
91
|
-
frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig;
|
|
92
|
-
/** Isolate main content area */
|
|
93
|
-
isolateMain?: boolean;
|
|
94
|
-
/** Convert Tailwind utility classes to markdown formatting */
|
|
95
|
-
tailwind?: boolean;
|
|
96
|
-
/**
|
|
97
|
-
* Extract elements matching CSS selectors during conversion.
|
|
98
|
-
* Each key is a CSS selector; the handler is called for every match.
|
|
99
|
-
*/
|
|
100
|
-
extraction?: Record<string, (element: ExtractedElement) => void>;
|
|
101
|
-
/**
|
|
102
|
-
* Declarative tag overrides for customizing tag behavior.
|
|
103
|
-
* String values act as aliases (e.g. `{ "x-heading": "h2" }` makes `<x-heading>` behave like `<h2>`).
|
|
104
|
-
* Object values override specific handler properties.
|
|
105
|
-
*/
|
|
106
|
-
tagOverrides?: Record<string, TagOverride | string>;
|
|
107
|
-
}
|
|
108
|
-
/**
|
|
109
|
-
* Shared engine options that work with both JS and Rust engines.
|
|
110
|
-
* This is the contract that `MarkdownEngine` methods accept.
|
|
111
|
-
*/
|
|
87
|
+
/** Cleanup rules. */
|
|
112
88
|
export interface CleanOptions {
|
|
113
89
|
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
114
90
|
urls?: boolean;
|
|
@@ -116,30 +92,78 @@ export interface CleanOptions {
|
|
|
116
92
|
fragments?: boolean;
|
|
117
93
|
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
118
94
|
emptyLinks?: boolean;
|
|
119
|
-
/** Collapse 3+ consecutive blank lines to 2 */
|
|
120
|
-
blankLines?: boolean;
|
|
121
95
|
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
122
96
|
redundantLinks?: boolean;
|
|
123
97
|
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
|
124
98
|
selfLinkHeadings?: boolean;
|
|
125
|
-
/**
|
|
99
|
+
/** Drop images with missing, empty, or whitespace-only alt text */
|
|
126
100
|
emptyImages?: boolean;
|
|
127
101
|
/** Drop links that produce no visible text: [](url) → nothing */
|
|
128
102
|
emptyLinkText?: boolean;
|
|
129
103
|
}
|
|
104
|
+
/** Converter state a `CleanPass` reads and rewrites. */
|
|
105
|
+
interface CleanTarget {
|
|
106
|
+
/** Markdown written so far, one entry per write. */
|
|
107
|
+
buffer: string[];
|
|
108
|
+
/** Open element count per tag id. */
|
|
109
|
+
depthMap: Uint16Array;
|
|
110
|
+
/** The last buffer entry the converter wrote. */
|
|
111
|
+
lastContentCache?: string;
|
|
112
|
+
options?: EngineOptions;
|
|
113
|
+
/** @internal */
|
|
114
|
+
outputPositions?: OutputPositions;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Cleanup for one conversion. The converter calls these hooks; they rewrite
|
|
118
|
+
* links after the converter writes them.
|
|
119
|
+
*/
|
|
120
|
+
interface CleanPass {
|
|
121
|
+
/** Output waits for the whole document, because `fragments` needs every heading. */
|
|
122
|
+
holdsOutput: boolean;
|
|
123
|
+
/** An element's enter output starts at `outputStart` in the buffer. */
|
|
124
|
+
enter: (element: ElementNode, outputStart: number) => void;
|
|
125
|
+
/** An element exits; called before its exit output. */
|
|
126
|
+
exit: (element: ElementNode) => void;
|
|
127
|
+
/** Rewrite the closing anchor to its text and skip its close. Returns true when it did. */
|
|
128
|
+
unwrap: (element: ElementNode) => boolean;
|
|
129
|
+
/** An element wrote its exit output `close` at `outputStart`. */
|
|
130
|
+
closed: (element: ElementNode, outputStart: number, close: string) => void;
|
|
131
|
+
/** The earliest buffer index a later hook may rewrite, or Infinity. */
|
|
132
|
+
held: () => number;
|
|
133
|
+
/** Apply the rules that need the whole document to the trimmed Markdown so far. */
|
|
134
|
+
finish: (markdown: string) => CleanView;
|
|
135
|
+
}
|
|
136
|
+
/** Markdown a `CleanPass` finished, with optional source position mapping. */
|
|
137
|
+
interface CleanView {
|
|
138
|
+
markdown: string;
|
|
139
|
+
/**
|
|
140
|
+
* Map a raw Markdown offset through this view's removal runs.
|
|
141
|
+
* @internal
|
|
142
|
+
*/
|
|
143
|
+
mapPosition?: (position: number) => number;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Cleanup rules and the pass that applies them.
|
|
147
|
+
* Create one with `clean()` from `@mdream/js/clean`.
|
|
148
|
+
*/
|
|
149
|
+
export interface Cleaner extends CleanOptions {
|
|
150
|
+
/** Start cleanup for one conversion. Returns nothing when no rule rewrites links. */
|
|
151
|
+
apply: (target: CleanTarget) => CleanPass | undefined;
|
|
152
|
+
/** Rewrite a resolved link or image URL. Set when `urls` is on. */
|
|
153
|
+
url?: (url: string) => string;
|
|
154
|
+
}
|
|
155
|
+
/** Core conversion options. */
|
|
130
156
|
export interface EngineOptions {
|
|
131
157
|
/**
|
|
132
158
|
* Origin URL for resolving relative image paths and internal links.
|
|
133
159
|
*/
|
|
134
160
|
origin?: string;
|
|
135
|
-
/**
|
|
136
|
-
|
|
137
|
-
*/
|
|
138
|
-
plugins?: BuiltinPlugins;
|
|
161
|
+
/** Declarative tag behavior overrides. */
|
|
162
|
+
tagOverrides?: Record<string, TagOverride | string>;
|
|
139
163
|
/**
|
|
140
164
|
* Clean up the markdown output. Pass `true` for all cleanup or an object
|
|
141
|
-
* to enable specific features.
|
|
142
|
-
*
|
|
165
|
+
* to enable specific features. The rules apply to the link and image nodes
|
|
166
|
+
* as they convert; `fragments` also needs `apply` from a `Cleaner`.
|
|
143
167
|
*/
|
|
144
168
|
clean?: boolean | CleanOptions;
|
|
145
169
|
/**
|
|
@@ -149,11 +173,6 @@ export interface EngineOptions {
|
|
|
149
173
|
* wrapping.
|
|
150
174
|
*/
|
|
151
175
|
wrapWidth?: number;
|
|
152
|
-
/**
|
|
153
|
-
* Output format. Defaults to `markdown`; use `text` for readable plain text,
|
|
154
|
-
* or `html` for allowlisted semantic HTML.
|
|
155
|
-
*/
|
|
156
|
-
format?: 'markdown' | 'text' | 'html';
|
|
157
176
|
}
|
|
158
177
|
export interface ElementNode extends Node {
|
|
159
178
|
/** Element tag name (for ELEMENT_NODE) */
|
|
@@ -168,6 +187,8 @@ export interface ElementNode extends Node {
|
|
|
168
187
|
depthMap: Uint16Array;
|
|
169
188
|
/** Plugin outputs collected during processing */
|
|
170
189
|
pluginOutput?: string[];
|
|
190
|
+
/** Whether this element's enter event wrote an inline wrapper. */
|
|
191
|
+
hasEnterOutput?: boolean;
|
|
171
192
|
}
|
|
172
193
|
export interface TextNode extends Node {
|
|
173
194
|
/** Text content (for TEXT_NODE) */
|
|
@@ -179,6 +200,13 @@ export interface TextNode extends Node {
|
|
|
179
200
|
};
|
|
180
201
|
/** Custom data added by plugins */
|
|
181
202
|
context?: PluginContext;
|
|
203
|
+
/** Follows an end tag that closed nothing, so it gets no separator. */
|
|
204
|
+
joinsPrevious?: boolean;
|
|
205
|
+
/**
|
|
206
|
+
* First text with no block ancestor: it drops its leading whitespace when
|
|
207
|
+
* the output is at the start of a line.
|
|
208
|
+
*/
|
|
209
|
+
trimsAtLineStart?: boolean;
|
|
182
210
|
}
|
|
183
211
|
/**
|
|
184
212
|
* Base DOM node interface
|
|
@@ -210,6 +238,7 @@ export interface Node {
|
|
|
210
238
|
/**
|
|
211
239
|
* State interface for HTML parsing and processing
|
|
212
240
|
* Contains parsing state that's maintained during HTML traversal
|
|
241
|
+
* @internal
|
|
213
242
|
*/
|
|
214
243
|
interface MdreamProcessingState {
|
|
215
244
|
/** Map of tag names to their current nesting depth - uses TypedArray for performance */
|
|
@@ -233,31 +262,32 @@ interface MdreamProcessingState {
|
|
|
233
262
|
isFirstTextInElement?: boolean;
|
|
234
263
|
/** Reference to the last processed text node - for context tracking */
|
|
235
264
|
lastTextNode?: Node;
|
|
236
|
-
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
237
|
-
inSingleQuote?: boolean;
|
|
238
|
-
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
239
|
-
inDoubleQuote?: boolean;
|
|
240
|
-
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
241
|
-
inBacktick?: boolean;
|
|
242
|
-
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
243
|
-
lastCharWasBackslash?: boolean;
|
|
244
265
|
/** Resolved plugin instances for efficient iteration */
|
|
245
266
|
resolvedPlugins?: TransformPlugin[];
|
|
246
267
|
/** Configuration options for conversion */
|
|
247
268
|
options?: EngineOptions;
|
|
248
269
|
}
|
|
249
270
|
/**
|
|
250
|
-
*
|
|
251
|
-
*
|
|
271
|
+
* Writable converter state. Plugin hooks see it as `PluginState`; tag
|
|
272
|
+
* handlers and the built-in plugins that write output read the rest.
|
|
273
|
+
* @internal
|
|
252
274
|
*/
|
|
253
|
-
|
|
275
|
+
interface MdreamRuntimeState extends Partial<MdreamProcessingState>, PluginState {
|
|
276
|
+
/** @internal */
|
|
277
|
+
outputPositions?: OutputPositions;
|
|
278
|
+
/** Active output format for format-aware plugins. */
|
|
279
|
+
outputFormat: OutputFormat;
|
|
280
|
+
/** Nesting depth of the current node. */
|
|
281
|
+
depth: number;
|
|
254
282
|
/** Number of newlines at end of most recent output */
|
|
255
283
|
lastNewLines?: number;
|
|
256
284
|
/** Configuration options for conversion */
|
|
257
|
-
options
|
|
285
|
+
options: EngineOptions;
|
|
258
286
|
/** Table processing state - specialized for Markdown tables */
|
|
259
287
|
tableRenderedTable?: boolean;
|
|
260
288
|
tableCurrentRowCells?: number;
|
|
289
|
+
/** A built-in row waits for its first cell before writing its opener. */
|
|
290
|
+
tableRowOpenerPending?: boolean;
|
|
261
291
|
tableColumnAlignments?: string[];
|
|
262
292
|
/** See MarkdownState for semantics. */
|
|
263
293
|
tableHeaderCells?: number;
|
|
@@ -281,8 +311,9 @@ export interface MdreamRuntimeState extends Partial<MdreamProcessingState> {
|
|
|
281
311
|
* <pre> fenced-code deferral (issue #97). See MarkdownState for semantics.
|
|
282
312
|
*/
|
|
283
313
|
preFencePending?: boolean;
|
|
314
|
+
preFencePendingDepth?: number;
|
|
284
315
|
preFenceLang?: string;
|
|
285
|
-
|
|
316
|
+
preFenceOwnerDepth?: number;
|
|
286
317
|
/** Number of default blockquotes currently buffered for line prefixing. */
|
|
287
318
|
bufferedBlockquoteDepth?: number;
|
|
288
319
|
/** Content-column prefix deferred after a list item rule. */
|
|
@@ -357,6 +388,8 @@ export interface TagHandler {
|
|
|
357
388
|
literalEnter?: boolean;
|
|
358
389
|
/** When true, the `exit` string is a user-supplied tagOverride, exempt from empty-pair cleanup. */
|
|
359
390
|
literalExit?: boolean;
|
|
391
|
+
/** Whether a declarative literal exit string writes output. */
|
|
392
|
+
literalExitHasOutput?: boolean;
|
|
360
393
|
/**
|
|
361
394
|
* Built-in tag id used by declarative string aliases.
|
|
362
395
|
* @internal
|
|
@@ -369,37 +402,18 @@ interface TailwindContext {
|
|
|
369
402
|
suffix?: string;
|
|
370
403
|
}
|
|
371
404
|
export interface PluginContext {
|
|
372
|
-
score?: number;
|
|
373
|
-
tagCount?: number;
|
|
374
|
-
linkTextLength?: number;
|
|
375
|
-
textLength?: number;
|
|
376
|
-
isHighLinkDensity?: boolean;
|
|
377
405
|
tailwind?: TailwindContext;
|
|
378
406
|
[key: string]: unknown;
|
|
379
407
|
}
|
|
380
|
-
/**
|
|
381
|
-
|
|
382
|
-
*/
|
|
383
|
-
export interface ExtractedElement {
|
|
384
|
-
/** The CSS selector that matched this element */
|
|
385
|
-
selector: string;
|
|
386
|
-
/** The HTML tag name */
|
|
387
|
-
tagName: string;
|
|
388
|
-
/** Accumulated text content of the element */
|
|
389
|
-
textContent: string;
|
|
390
|
-
/** HTML attributes of the element */
|
|
391
|
-
attributes: Record<string, string>;
|
|
392
|
-
}
|
|
393
|
-
/**
|
|
394
|
-
* Top-level options for the mdream JS engine.
|
|
395
|
-
* Extends the shared `EngineOptions` with JS-specific concerns.
|
|
396
|
-
*/
|
|
397
|
-
export interface MdreamOptions extends EngineOptions {
|
|
408
|
+
/** Top-level options for the mdream JavaScript engine. */
|
|
409
|
+
export interface MdreamOptions extends Omit<EngineOptions, 'clean'> {
|
|
398
410
|
/**
|
|
399
|
-
*
|
|
400
|
-
*
|
|
411
|
+
* Cleanup rules from `clean()` in `@mdream/js/clean`. Import it only when
|
|
412
|
+
* you use it, so the cleanup pass stays out of other bundles.
|
|
401
413
|
*/
|
|
402
|
-
|
|
414
|
+
clean?: Cleaner;
|
|
415
|
+
/** Explicit plugins, applied in array order. */
|
|
416
|
+
plugins?: Plugin[];
|
|
403
417
|
}
|
|
404
418
|
/**
|
|
405
419
|
* Markdown chunk with content and metadata
|
|
@@ -427,13 +441,13 @@ export interface MarkdownChunk {
|
|
|
427
441
|
* Options for HTML to Markdown chunking
|
|
428
442
|
* Extends EngineOptions with chunking-specific settings
|
|
429
443
|
*/
|
|
430
|
-
export interface SplitterOptions extends
|
|
444
|
+
export interface SplitterOptions extends MdreamOptions {
|
|
431
445
|
/**
|
|
432
|
-
*
|
|
433
|
-
* @example [
|
|
434
|
-
* @default [
|
|
446
|
+
* Heading levels that start a new chunk, from 1 (`<h1>`) to 6 (`<h6>`).
|
|
447
|
+
* @example [1, 2]
|
|
448
|
+
* @default [2, 3, 4, 5, 6]
|
|
435
449
|
*/
|
|
436
|
-
headersToSplitOn?:
|
|
450
|
+
headersToSplitOn?: (1 | 2 | 3 | 4 | 5 | 6)[];
|
|
437
451
|
/**
|
|
438
452
|
* Return each line as individual chunk
|
|
439
453
|
* @default false
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { escapeHtml, isSafeHtmlUrl } from "./utils.mjs";
|
|
2
|
+
const URL_SCHEME_RE = /^[A-Z][\dA-Z+.-]*:/i;
|
|
3
|
+
const SLASH_CHAR = 47;
|
|
4
|
+
function pathEnd(value, from) {
|
|
5
|
+
const query = value.indexOf("?", from);
|
|
6
|
+
const fragment = value.indexOf("#", from);
|
|
7
|
+
if (query === -1) return fragment === -1 ? value.length : fragment;
|
|
8
|
+
return fragment === -1 ? query : Math.min(query, fragment);
|
|
9
|
+
}
|
|
10
|
+
function hasDotSegment(path) {
|
|
11
|
+
return path.includes(".") && (path === "." || path === ".." || path.startsWith("./") || path.startsWith("../") || path.includes("/./") || path.includes("/../") || path.endsWith("/.") || path.endsWith("/.."));
|
|
12
|
+
}
|
|
13
|
+
function mergeDotSegments(directory, rest) {
|
|
14
|
+
let output = directory;
|
|
15
|
+
let index = 0;
|
|
16
|
+
for (;;) {
|
|
17
|
+
let end = rest.indexOf("/", index);
|
|
18
|
+
const last = end === -1;
|
|
19
|
+
if (last) end = rest.length;
|
|
20
|
+
const segment = rest.slice(index, end);
|
|
21
|
+
if (segment === "..") output = output.slice(0, output.lastIndexOf("/", output.length - 2) + 1);
|
|
22
|
+
else if (segment !== ".") output += last ? segment : `${segment}/`;
|
|
23
|
+
if (last) return output;
|
|
24
|
+
index = end + 1;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
let baseOrigin;
|
|
28
|
+
let baseRoot = "";
|
|
29
|
+
let basePath = "";
|
|
30
|
+
let baseDirectory = "";
|
|
31
|
+
function loadBase(origin) {
|
|
32
|
+
if (origin !== baseOrigin) {
|
|
33
|
+
baseOrigin = origin;
|
|
34
|
+
baseRoot = "";
|
|
35
|
+
const authority = origin.indexOf("://");
|
|
36
|
+
if (authority !== -1) {
|
|
37
|
+
const end = pathEnd(origin, authority + 3);
|
|
38
|
+
const slash = origin.indexOf("/", authority + 3);
|
|
39
|
+
const rootEnd = slash === -1 || slash > end ? end : slash;
|
|
40
|
+
baseRoot = origin.slice(0, rootEnd);
|
|
41
|
+
basePath = origin.slice(rootEnd, end);
|
|
42
|
+
baseDirectory = basePath.slice(0, basePath.lastIndexOf("/") + 1) || "/";
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return baseRoot !== "";
|
|
46
|
+
}
|
|
47
|
+
function resolveAgainstBase(origin, url) {
|
|
48
|
+
if (!loadBase(origin)) {
|
|
49
|
+
let prefix = origin;
|
|
50
|
+
while (prefix.endsWith("/")) prefix = prefix.slice(0, -1);
|
|
51
|
+
const path = url.startsWith("./") ? url.slice(2) : url;
|
|
52
|
+
return `${prefix}${path.charCodeAt(0) === SLASH_CHAR ? "" : "/"}${path}`;
|
|
53
|
+
}
|
|
54
|
+
const end = pathEnd(url, 0);
|
|
55
|
+
const path = url.slice(0, end);
|
|
56
|
+
const suffix = end === url.length ? "" : url.slice(end);
|
|
57
|
+
if (!path) return `${baseRoot}${basePath}${suffix}`;
|
|
58
|
+
const dot = hasDotSegment(path);
|
|
59
|
+
if (path.charCodeAt(0) === SLASH_CHAR) return `${baseRoot}${dot ? mergeDotSegments("/", path.slice(1)) : path}${suffix}`;
|
|
60
|
+
return `${baseRoot}${dot ? mergeDotSegments(baseDirectory, path) : baseDirectory + path}${suffix}`;
|
|
61
|
+
}
|
|
62
|
+
function resolveUrl(url, origin, clean) {
|
|
63
|
+
if (!url || url[0] === "#") return url;
|
|
64
|
+
const isSlash = url.charCodeAt(0) === SLASH_CHAR;
|
|
65
|
+
let resolved = url;
|
|
66
|
+
if (isSlash && url.charCodeAt(1) === SLASH_CHAR) {
|
|
67
|
+
let scheme = "https";
|
|
68
|
+
if (origin && loadBase(origin)) scheme = baseRoot.slice(0, baseRoot.indexOf(":"));
|
|
69
|
+
resolved = `${scheme}:${url}`;
|
|
70
|
+
} else if (origin && (isSlash || !URL_SCHEME_RE.test(url))) resolved = resolveAgainstBase(origin, url);
|
|
71
|
+
const rewrite = clean?.url;
|
|
72
|
+
return rewrite ? rewrite(resolved) : resolved;
|
|
73
|
+
}
|
|
74
|
+
function safeAnchorOutput(node, options, entering, protectMarkdown = false) {
|
|
75
|
+
const href = node.attributes?.href;
|
|
76
|
+
if (!href || !isSafeHtmlUrl(href)) return;
|
|
77
|
+
const resolved = resolveUrl(href, options?.origin, options?.clean);
|
|
78
|
+
if (!isSafeHtmlUrl(resolved)) return;
|
|
79
|
+
if (!entering) return "</a>";
|
|
80
|
+
return `<a href="${escapeHtml(resolved, true, protectMarkdown)}"${node.attributes?.title === void 0 ? "" : ` title="${escapeHtml(node.attributes.title, true, protectMarkdown)}"`}>`;
|
|
81
|
+
}
|
|
82
|
+
export { resolveUrl, safeAnchorOutput };
|