@mdream/js 1.4.1 → 1.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -7
- package/dist/_chunks/const.mjs +2 -2
- package/dist/_chunks/index.d.mts +1 -0
- package/dist/_chunks/markdown-processor.mjs +464 -0
- package/dist/_chunks/parse.mjs +757 -473
- package/dist/_chunks/plugins.mjs +18 -19
- package/dist/_chunks/resolve-plugins.mjs +1 -341
- package/dist/_chunks/src.mjs +5 -36
- package/dist/_chunks/stream.mjs +35 -0
- package/dist/_chunks/types.d.mts +25 -6
- package/dist/cli.mjs +9 -2
- package/dist/core.d.mts +16 -0
- package/dist/core.mjs +26 -0
- package/dist/parse.d.mts +13 -8
- package/dist/plugins.d.mts +6 -5
- package/dist/splitter.mjs +11 -2
- package/package.json +12 -2
package/dist/core.mjs
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { buildTagOverrideHandlers } from "./_chunks/parse.mjs";
|
|
2
|
+
import { createMarkdownProcessor } from "./_chunks/markdown-processor.mjs";
|
|
3
|
+
import { streamHtmlToMarkdown as streamHtmlToMarkdown$1 } from "./_chunks/stream.mjs";
|
|
4
|
+
function resolveHooks(options) {
|
|
5
|
+
const plugins = Array.isArray(options.plugins) ? options.plugins : void 0;
|
|
6
|
+
const hooks = options.hooks?.length ? options.hooks : void 0;
|
|
7
|
+
if (!plugins?.length) return hooks || [];
|
|
8
|
+
if (!hooks) return plugins;
|
|
9
|
+
return [...plugins, ...hooks];
|
|
10
|
+
}
|
|
11
|
+
function resolveTagOverrides(options) {
|
|
12
|
+
return !Array.isArray(options.plugins) ? options.plugins?.tagOverrides : void 0;
|
|
13
|
+
}
|
|
14
|
+
function htmlToMarkdown(html, options = {}) {
|
|
15
|
+
const plugins = resolveHooks(options);
|
|
16
|
+
const tagOverrides = resolveTagOverrides(options);
|
|
17
|
+
const processor = createMarkdownProcessor(options, plugins, tagOverrides ? buildTagOverrideHandlers(tagOverrides) : void 0);
|
|
18
|
+
processor.processHtml(html);
|
|
19
|
+
return processor.getMarkdown();
|
|
20
|
+
}
|
|
21
|
+
function streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
22
|
+
const plugins = resolveHooks(options);
|
|
23
|
+
const tagOverrides = resolveTagOverrides(options);
|
|
24
|
+
return streamHtmlToMarkdown$1(htmlStream, options, plugins, tagOverrides ? buildTagOverrideHandlers(tagOverrides) : void 0);
|
|
25
|
+
}
|
|
26
|
+
export { htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/parse.d.mts
CHANGED
|
@@ -21,6 +21,8 @@ interface ParseState {
|
|
|
21
21
|
depthMap: Uint8Array;
|
|
22
22
|
/** Current overall nesting depth */
|
|
23
23
|
depth: number;
|
|
24
|
+
/** Whether parsing stopped after reaching the practical browser depth limit. */
|
|
25
|
+
depthLimitReached?: boolean;
|
|
24
26
|
/** Currently processing element node */
|
|
25
27
|
currentNode?: ElementNode | null;
|
|
26
28
|
/** Whether current content contains HTML entities that need decoding */
|
|
@@ -35,25 +37,28 @@ interface ParseState {
|
|
|
35
37
|
justClosedTag?: boolean;
|
|
36
38
|
/** Whether the next text node is the first in its element - for whitespace trimming */
|
|
37
39
|
isFirstTextInElement?: boolean;
|
|
40
|
+
/** Persisted HTML tokenizer state while streaming a built-in script body. */
|
|
41
|
+
scriptDataState?: number;
|
|
42
|
+
/** Consumed script chunks retained for extraction/plugin text events. */
|
|
43
|
+
scriptTextChunks?: string[];
|
|
38
44
|
/** Reference to the last processed text node - for context tracking */
|
|
39
45
|
lastTextNode?: Node;
|
|
40
|
-
/**
|
|
46
|
+
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
41
47
|
inSingleQuote?: boolean;
|
|
48
|
+
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
42
49
|
inDoubleQuote?: boolean;
|
|
50
|
+
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
43
51
|
inBacktick?: boolean;
|
|
44
|
-
/**
|
|
45
|
-
* Whether the current non-nesting tag is quote-aware rawtext (script/style only).
|
|
46
|
-
* For these, a `</script>`/`</style>` inside a JS/CSS string literal must not close
|
|
47
|
-
* the tag. Other non-nesting tags (template, textarea, ...) hold literal HTML/text
|
|
48
|
-
* where apostrophes are not string delimiters, so their closing tag always wins.
|
|
49
|
-
*/
|
|
52
|
+
/** @deprecated Rawtext parsing is no longer quote-aware. Retained for source compatibility. */
|
|
50
53
|
inRawTextQuoteAware?: boolean;
|
|
51
|
-
/**
|
|
54
|
+
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
52
55
|
lastCharWasBackslash?: boolean;
|
|
53
56
|
/** Resolved plugin instances for event processing */
|
|
54
57
|
resolvedPlugins?: TransformPlugin[];
|
|
55
58
|
/** Tag override handlers built from declarative tagOverrides config */
|
|
56
59
|
tagOverrideHandlers?: Map<string, TagHandler>;
|
|
60
|
+
/** Whether emitted text should skip Markdown-only escaping */
|
|
61
|
+
plainText?: boolean;
|
|
57
62
|
}
|
|
58
63
|
interface ParseResult {
|
|
59
64
|
events: NodeEvent[];
|
package/dist/plugins.d.mts
CHANGED
|
@@ -3,9 +3,7 @@ import { createPlugin } from "./_chunks/index.mjs";
|
|
|
3
3
|
interface ExtractedElement extends ElementNode {
|
|
4
4
|
textContent: string;
|
|
5
5
|
}
|
|
6
|
-
/**
|
|
7
|
-
* @deprecated Use `plugins.extraction` config for declarative extraction that works with both JS and Rust engines.
|
|
8
|
-
*/
|
|
6
|
+
/** Extract matching elements through the composable JavaScript plugin interface. */
|
|
9
7
|
declare function extractionPlugin(selectors: Record<string, (element: ExtractedElement, state: MdreamRuntimeState) => void>): TransformPlugin;
|
|
10
8
|
/**
|
|
11
9
|
* Extraction collector for `plugins.extraction` config.
|
|
@@ -30,8 +28,11 @@ declare function extractionCollectorPlugin(config: Record<string, (element: Extr
|
|
|
30
28
|
* withQuerySelectorPlugin({ exclude: ['nav', '#sidebar', '.footer'] })
|
|
31
29
|
*/
|
|
32
30
|
declare function filterPlugin(options?: {
|
|
33
|
-
/** CSS selectors, tag names, or TAG_* constants for elements to include (all others will be excluded) */
|
|
34
|
-
|
|
31
|
+
/** CSS selectors, tag names, or TAG_* constants for elements to include (all others will be excluded) */
|
|
32
|
+
include?: (string | number)[];
|
|
33
|
+
/** CSS selectors, tag names, or TAG_* constants for elements to exclude */
|
|
34
|
+
exclude?: (string | number)[];
|
|
35
|
+
/** Whether to also process the children of matching elements */
|
|
35
36
|
processChildren?: boolean;
|
|
36
37
|
keepAbsolute?: boolean;
|
|
37
38
|
}): TransformPlugin;
|
package/dist/splitter.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import "./_chunks/const.mjs";
|
|
2
2
|
import { parseHtmlStream } from "./_chunks/parse.mjs";
|
|
3
|
-
import { createMarkdownProcessor, processPluginsForEvent
|
|
3
|
+
import { createMarkdownProcessor, processPluginsForEvent } from "./_chunks/markdown-processor.mjs";
|
|
4
|
+
import { resolvePlugins } from "./_chunks/resolve-plugins.mjs";
|
|
4
5
|
const MARKDOWN_HEADER_LINE_RE = /^#{1,6}\s+/;
|
|
5
6
|
const NEWLINE_RE = /\n/g;
|
|
6
7
|
const DEFAULT_HEADERS_TO_SPLIT_ON = [
|
|
@@ -93,11 +94,19 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
|
|
|
93
94
|
resolvedPlugins: opts.resolvedPlugins
|
|
94
95
|
};
|
|
95
96
|
const eventBuffer = [];
|
|
97
|
+
const processResolvedEvent = opts.resolvedPlugins?.length ? (event) => processPluginsForEvent(event, opts.resolvedPlugins, processor.state, processor.processEvent) : processor.processEvent;
|
|
96
98
|
parseHtmlStream(html, parseState, (event) => {
|
|
97
99
|
eventBuffer.push(event);
|
|
98
100
|
});
|
|
99
101
|
for (const event of eventBuffer) {
|
|
100
102
|
const { type: eventType, node } = event;
|
|
103
|
+
const eventDepthMap = node.type === 1 ? node.depthMap : node.parent?.depthMap;
|
|
104
|
+
if (eventDepthMap) processor.state.depthMap.set(eventDepthMap);
|
|
105
|
+
else processor.state.depthMap.fill(0);
|
|
106
|
+
if (eventType === 1 && node.type === 1) {
|
|
107
|
+
const tagId = node.tagId;
|
|
108
|
+
if (tagId !== void 0 && tagId >= 0 && tagId < processor.state.depthMap.length) processor.state.depthMap[tagId] = Math.max(0, processor.state.depthMap[tagId] - 1);
|
|
109
|
+
}
|
|
101
110
|
if (node.type === 1) {
|
|
102
111
|
const element = node;
|
|
103
112
|
const tagId = element.tagId;
|
|
@@ -128,7 +137,7 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
|
|
|
128
137
|
if (tagId === 13 && eventType === 0) yield* flushChunk();
|
|
129
138
|
}
|
|
130
139
|
if (collectingHeaderText && node.type === 2) currentHeaderText += node.value;
|
|
131
|
-
|
|
140
|
+
processResolvedEvent(event);
|
|
132
141
|
if (!opts.returnEachLine) {
|
|
133
142
|
const currentMd = getCurrentMarkdown(processor.state);
|
|
134
143
|
if (opts.lengthFunction(currentMd.slice(lastChunkEndPosition)) > opts.chunkSize) {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/js",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.5.3",
|
|
5
5
|
"description": "JavaScript HTML-to-Markdown engine for mdream. Escape hatch for hooks and edge runtimes.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
"url": "https://github.com/harlan-zw/mdream",
|
|
15
15
|
"directory": "packages/js"
|
|
16
16
|
},
|
|
17
|
+
"sideEffects": false,
|
|
17
18
|
"exports": {
|
|
18
19
|
".": {
|
|
19
20
|
"types": "./dist/index.d.mts",
|
|
@@ -23,6 +24,14 @@
|
|
|
23
24
|
},
|
|
24
25
|
"default": "./dist/index.mjs"
|
|
25
26
|
},
|
|
27
|
+
"./core": {
|
|
28
|
+
"types": "./dist/core.d.mts",
|
|
29
|
+
"import": {
|
|
30
|
+
"types": "./dist/core.d.mts",
|
|
31
|
+
"default": "./dist/core.mjs"
|
|
32
|
+
},
|
|
33
|
+
"default": "./dist/core.mjs"
|
|
34
|
+
},
|
|
26
35
|
"./negotiate": {
|
|
27
36
|
"types": "./dist/negotiate.d.mts",
|
|
28
37
|
"import": {
|
|
@@ -88,6 +97,7 @@
|
|
|
88
97
|
"scripts": {
|
|
89
98
|
"build": "obuild",
|
|
90
99
|
"typecheck": "tsc --noEmit",
|
|
91
|
-
"dev:prepare": "obuild --stub"
|
|
100
|
+
"dev:prepare": "obuild --stub",
|
|
101
|
+
"test:attw": "attw --pack --profile esm-only"
|
|
92
102
|
}
|
|
93
103
|
}
|