mdream 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -9
- package/dist/_chunks/const.mjs +110 -228
- package/dist/_chunks/extraction.mjs +24 -1
- package/dist/_chunks/markdown-processor.mjs +256 -165
- package/dist/_chunks/plugin.mjs +7 -0
- package/dist/_chunks/{tailwind.mjs → plugins.mjs} +109 -26
- package/dist/_chunks/{stream.mjs → src.mjs} +16 -1
- package/dist/cli.mjs +7 -1
- package/dist/iife.js +3 -3
- package/dist/index.mjs +3 -8
- package/dist/llms-txt.mjs +91 -5
- package/dist/negotiate.d.mts +26 -0
- package/dist/negotiate.mjs +92 -0
- package/dist/plugins.mjs +2 -1
- package/dist/preset/minimal.mjs +28 -18
- package/dist/splitter.mjs +34 -19
- package/package.json +10 -2
package/README.md
CHANGED
|
@@ -545,12 +545,3 @@ Custom notes
|
|
|
545
545
|
|
|
546
546
|
Licensed under the [MIT license](https://github.com/harlan-zw/mdream/blob/main/LICENSE.md).
|
|
547
547
|
|
|
548
|
-
<!-- Badges -->
|
|
549
|
-
[npm-version-src]: https://img.shields.io/npm/v/mdream/latest.svg?style=flat&colorA=18181B&colorB=4C9BE0
|
|
550
|
-
[npm-version-href]: https://npmjs.com/package/mdream
|
|
551
|
-
|
|
552
|
-
[npm-downloads-src]: https://img.shields.io/npm/dm/mdream.svg?style=flat&colorA=18181B&colorB=4C9BE0
|
|
553
|
-
[npm-downloads-href]: https://npmjs.com/package/mdream
|
|
554
|
-
|
|
555
|
-
[license-src]: https://img.shields.io/github/license/harlan-zw/mdream.svg?style=flat&colorA=18181B&colorB=4C9BE0
|
|
556
|
-
[license-href]: https://github.com/harlan-zw/mdream/blob/main/LICENSE.md
|
package/dist/_chunks/const.mjs
CHANGED
|
@@ -1,112 +1,3 @@
|
|
|
1
|
-
const TAG_HTML = 0;
|
|
2
|
-
const TAG_HEAD = 1;
|
|
3
|
-
const TAG_DETAILS = 2;
|
|
4
|
-
const TAG_SUMMARY = 3;
|
|
5
|
-
const TAG_TITLE = 4;
|
|
6
|
-
const TAG_META = 5;
|
|
7
|
-
const TAG_BR = 6;
|
|
8
|
-
const TAG_H1 = 7;
|
|
9
|
-
const TAG_H2 = 8;
|
|
10
|
-
const TAG_H3 = 9;
|
|
11
|
-
const TAG_H4 = 10;
|
|
12
|
-
const TAG_H5 = 11;
|
|
13
|
-
const TAG_H6 = 12;
|
|
14
|
-
const TAG_HR = 13;
|
|
15
|
-
const TAG_STRONG = 14;
|
|
16
|
-
const TAG_B = 15;
|
|
17
|
-
const TAG_EM = 16;
|
|
18
|
-
const TAG_I = 17;
|
|
19
|
-
const TAG_DEL = 18;
|
|
20
|
-
const TAG_SUB = 19;
|
|
21
|
-
const TAG_SUP = 20;
|
|
22
|
-
const TAG_INS = 21;
|
|
23
|
-
const TAG_BLOCKQUOTE = 22;
|
|
24
|
-
const TAG_CODE = 23;
|
|
25
|
-
const TAG_UL = 24;
|
|
26
|
-
const TAG_LI = 25;
|
|
27
|
-
const TAG_A = 26;
|
|
28
|
-
const TAG_IMG = 27;
|
|
29
|
-
const TAG_TABLE = 28;
|
|
30
|
-
const TAG_THEAD = 29;
|
|
31
|
-
const TAG_TR = 30;
|
|
32
|
-
const TAG_TH = 31;
|
|
33
|
-
const TAG_TD = 32;
|
|
34
|
-
const TAG_OL = 33;
|
|
35
|
-
const TAG_PRE = 34;
|
|
36
|
-
const TAG_P = 35;
|
|
37
|
-
const TAG_DIV = 36;
|
|
38
|
-
const TAG_SPAN = 37;
|
|
39
|
-
const TAG_TBODY = 38;
|
|
40
|
-
const TAG_TFOOT = 39;
|
|
41
|
-
const TAG_FORM = 40;
|
|
42
|
-
const TAG_NAV = 41;
|
|
43
|
-
const TAG_LABEL = 42;
|
|
44
|
-
const TAG_BUTTON = 43;
|
|
45
|
-
const TAG_BODY = 44;
|
|
46
|
-
const TAG_CENTER = 45;
|
|
47
|
-
const TAG_KBD = 46;
|
|
48
|
-
const TAG_FOOTER = 47;
|
|
49
|
-
const TAG_PATH = 48;
|
|
50
|
-
const TAG_SVG = 49;
|
|
51
|
-
const TAG_ARTICLE = 50;
|
|
52
|
-
const TAG_SECTION = 51;
|
|
53
|
-
const TAG_SCRIPT = 52;
|
|
54
|
-
const TAG_STYLE = 53;
|
|
55
|
-
const TAG_LINK = 54;
|
|
56
|
-
const TAG_AREA = 55;
|
|
57
|
-
const TAG_BASE = 56;
|
|
58
|
-
const TAG_COL = 57;
|
|
59
|
-
const TAG_EMBED = 58;
|
|
60
|
-
const TAG_INPUT = 59;
|
|
61
|
-
const TAG_KEYGEN = 60;
|
|
62
|
-
const TAG_PARAM = 61;
|
|
63
|
-
const TAG_SOURCE = 62;
|
|
64
|
-
const TAG_TRACK = 63;
|
|
65
|
-
const TAG_WBR = 64;
|
|
66
|
-
const TAG_SELECT = 65;
|
|
67
|
-
const TAG_TEXTAREA = 66;
|
|
68
|
-
const TAG_OPTION = 67;
|
|
69
|
-
const TAG_FIELDSET = 68;
|
|
70
|
-
const TAG_LEGEND = 69;
|
|
71
|
-
const TAG_AUDIO = 70;
|
|
72
|
-
const TAG_VIDEO = 71;
|
|
73
|
-
const TAG_CANVAS = 72;
|
|
74
|
-
const TAG_IFRAME = 73;
|
|
75
|
-
const TAG_MAP = 74;
|
|
76
|
-
const TAG_DIALOG = 75;
|
|
77
|
-
const TAG_METER = 76;
|
|
78
|
-
const TAG_PROGRESS = 77;
|
|
79
|
-
const TAG_TEMPLATE = 78;
|
|
80
|
-
const TAG_ABBR = 79;
|
|
81
|
-
const TAG_MARK = 80;
|
|
82
|
-
const TAG_Q = 81;
|
|
83
|
-
const TAG_SAMP = 82;
|
|
84
|
-
const TAG_SMALL = 83;
|
|
85
|
-
const TAG_NOSCRIPT = 84;
|
|
86
|
-
const TAG_NOFRAMES = 85;
|
|
87
|
-
const TAG_XMP = 86;
|
|
88
|
-
const TAG_PLAINTEXT = 87;
|
|
89
|
-
const TAG_ASIDE = 88;
|
|
90
|
-
const TAG_U = 89;
|
|
91
|
-
const TAG_CITE = 90;
|
|
92
|
-
const TAG_DFN = 91;
|
|
93
|
-
const TAG_VAR = 92;
|
|
94
|
-
const TAG_TIME = 93;
|
|
95
|
-
const TAG_BDO = 94;
|
|
96
|
-
const TAG_RUBY = 95;
|
|
97
|
-
const TAG_RT = 96;
|
|
98
|
-
const TAG_RP = 97;
|
|
99
|
-
const TAG_DD = 98;
|
|
100
|
-
const TAG_DT = 99;
|
|
101
|
-
const TAG_ADDRESS = 100;
|
|
102
|
-
const TAG_DL = 101;
|
|
103
|
-
const TAG_FIGURE = 102;
|
|
104
|
-
const TAG_OBJECT = 103;
|
|
105
|
-
const TAG_MAIN = 104;
|
|
106
|
-
const TAG_HEADER = 105;
|
|
107
|
-
const TAG_FIGCAPTION = 106;
|
|
108
|
-
const TAG_CAPTION = 107;
|
|
109
|
-
const MAX_TAG_ID = 108;
|
|
110
1
|
const HTML_ENTITIES = {
|
|
111
2
|
"&": "&",
|
|
112
3
|
"<": "<",
|
|
@@ -116,129 +7,120 @@ const HTML_ENTITIES = {
|
|
|
116
7
|
"'": "'",
|
|
117
8
|
" ": " "
|
|
118
9
|
};
|
|
119
|
-
const ELEMENT_NODE = 1;
|
|
120
|
-
const TEXT_NODE = 2;
|
|
121
|
-
const NodeEventEnter = 0;
|
|
122
|
-
const NodeEventExit = 1;
|
|
123
10
|
const TagIdMap = {
|
|
124
|
-
html:
|
|
125
|
-
head:
|
|
126
|
-
details:
|
|
127
|
-
summary:
|
|
128
|
-
title:
|
|
129
|
-
meta:
|
|
130
|
-
br:
|
|
131
|
-
h1:
|
|
132
|
-
h2:
|
|
133
|
-
h3:
|
|
134
|
-
h4:
|
|
135
|
-
h5:
|
|
136
|
-
h6:
|
|
137
|
-
hr:
|
|
138
|
-
strong:
|
|
139
|
-
b:
|
|
140
|
-
em:
|
|
141
|
-
i:
|
|
142
|
-
del:
|
|
143
|
-
sub:
|
|
144
|
-
sup:
|
|
145
|
-
ins:
|
|
146
|
-
blockquote:
|
|
147
|
-
code:
|
|
148
|
-
ul:
|
|
149
|
-
li:
|
|
150
|
-
a:
|
|
151
|
-
img:
|
|
152
|
-
table:
|
|
153
|
-
thead:
|
|
154
|
-
tr:
|
|
155
|
-
th:
|
|
156
|
-
td:
|
|
157
|
-
ol:
|
|
158
|
-
pre:
|
|
159
|
-
p:
|
|
160
|
-
div:
|
|
161
|
-
span:
|
|
162
|
-
tbody:
|
|
163
|
-
tfoot:
|
|
164
|
-
form:
|
|
165
|
-
nav:
|
|
166
|
-
label:
|
|
167
|
-
button:
|
|
168
|
-
body:
|
|
169
|
-
center:
|
|
170
|
-
kbd:
|
|
171
|
-
footer:
|
|
172
|
-
path:
|
|
173
|
-
svg:
|
|
174
|
-
article:
|
|
175
|
-
section:
|
|
176
|
-
script:
|
|
177
|
-
style:
|
|
178
|
-
link:
|
|
179
|
-
area:
|
|
180
|
-
base:
|
|
181
|
-
col:
|
|
182
|
-
embed:
|
|
183
|
-
input:
|
|
184
|
-
keygen:
|
|
185
|
-
param:
|
|
186
|
-
source:
|
|
187
|
-
track:
|
|
188
|
-
wbr:
|
|
189
|
-
select:
|
|
190
|
-
textarea:
|
|
191
|
-
option:
|
|
192
|
-
fieldset:
|
|
193
|
-
legend:
|
|
194
|
-
audio:
|
|
195
|
-
video:
|
|
196
|
-
canvas:
|
|
197
|
-
iframe:
|
|
198
|
-
map:
|
|
199
|
-
dialog:
|
|
200
|
-
meter:
|
|
201
|
-
progress:
|
|
202
|
-
template:
|
|
203
|
-
abbr:
|
|
204
|
-
mark:
|
|
205
|
-
q:
|
|
206
|
-
samp:
|
|
207
|
-
small:
|
|
208
|
-
noscript:
|
|
209
|
-
noframes:
|
|
210
|
-
xmp:
|
|
211
|
-
plaintext:
|
|
212
|
-
aside:
|
|
213
|
-
u:
|
|
214
|
-
cite:
|
|
215
|
-
dfn:
|
|
216
|
-
var:
|
|
217
|
-
time:
|
|
218
|
-
bdo:
|
|
219
|
-
ruby:
|
|
220
|
-
rt:
|
|
221
|
-
rp:
|
|
222
|
-
dd:
|
|
223
|
-
dt:
|
|
224
|
-
dl:
|
|
225
|
-
address:
|
|
226
|
-
figure:
|
|
227
|
-
object:
|
|
228
|
-
main:
|
|
229
|
-
header:
|
|
230
|
-
figcaption:
|
|
231
|
-
caption:
|
|
11
|
+
html: 0,
|
|
12
|
+
head: 1,
|
|
13
|
+
details: 2,
|
|
14
|
+
summary: 3,
|
|
15
|
+
title: 4,
|
|
16
|
+
meta: 5,
|
|
17
|
+
br: 6,
|
|
18
|
+
h1: 7,
|
|
19
|
+
h2: 8,
|
|
20
|
+
h3: 9,
|
|
21
|
+
h4: 10,
|
|
22
|
+
h5: 11,
|
|
23
|
+
h6: 12,
|
|
24
|
+
hr: 13,
|
|
25
|
+
strong: 14,
|
|
26
|
+
b: 15,
|
|
27
|
+
em: 16,
|
|
28
|
+
i: 17,
|
|
29
|
+
del: 18,
|
|
30
|
+
sub: 19,
|
|
31
|
+
sup: 20,
|
|
32
|
+
ins: 21,
|
|
33
|
+
blockquote: 22,
|
|
34
|
+
code: 23,
|
|
35
|
+
ul: 24,
|
|
36
|
+
li: 25,
|
|
37
|
+
a: 26,
|
|
38
|
+
img: 27,
|
|
39
|
+
table: 28,
|
|
40
|
+
thead: 29,
|
|
41
|
+
tr: 30,
|
|
42
|
+
th: 31,
|
|
43
|
+
td: 32,
|
|
44
|
+
ol: 33,
|
|
45
|
+
pre: 34,
|
|
46
|
+
p: 35,
|
|
47
|
+
div: 36,
|
|
48
|
+
span: 37,
|
|
49
|
+
tbody: 38,
|
|
50
|
+
tfoot: 39,
|
|
51
|
+
form: 40,
|
|
52
|
+
nav: 41,
|
|
53
|
+
label: 42,
|
|
54
|
+
button: 43,
|
|
55
|
+
body: 44,
|
|
56
|
+
center: 45,
|
|
57
|
+
kbd: 46,
|
|
58
|
+
footer: 47,
|
|
59
|
+
path: 48,
|
|
60
|
+
svg: 49,
|
|
61
|
+
article: 50,
|
|
62
|
+
section: 51,
|
|
63
|
+
script: 52,
|
|
64
|
+
style: 53,
|
|
65
|
+
link: 54,
|
|
66
|
+
area: 55,
|
|
67
|
+
base: 56,
|
|
68
|
+
col: 57,
|
|
69
|
+
embed: 58,
|
|
70
|
+
input: 59,
|
|
71
|
+
keygen: 60,
|
|
72
|
+
param: 61,
|
|
73
|
+
source: 62,
|
|
74
|
+
track: 63,
|
|
75
|
+
wbr: 64,
|
|
76
|
+
select: 65,
|
|
77
|
+
textarea: 66,
|
|
78
|
+
option: 67,
|
|
79
|
+
fieldset: 68,
|
|
80
|
+
legend: 69,
|
|
81
|
+
audio: 70,
|
|
82
|
+
video: 71,
|
|
83
|
+
canvas: 72,
|
|
84
|
+
iframe: 73,
|
|
85
|
+
map: 74,
|
|
86
|
+
dialog: 75,
|
|
87
|
+
meter: 76,
|
|
88
|
+
progress: 77,
|
|
89
|
+
template: 78,
|
|
90
|
+
abbr: 79,
|
|
91
|
+
mark: 80,
|
|
92
|
+
q: 81,
|
|
93
|
+
samp: 82,
|
|
94
|
+
small: 83,
|
|
95
|
+
noscript: 84,
|
|
96
|
+
noframes: 85,
|
|
97
|
+
xmp: 86,
|
|
98
|
+
plaintext: 87,
|
|
99
|
+
aside: 88,
|
|
100
|
+
u: 89,
|
|
101
|
+
cite: 90,
|
|
102
|
+
dfn: 91,
|
|
103
|
+
var: 92,
|
|
104
|
+
time: 93,
|
|
105
|
+
bdo: 94,
|
|
106
|
+
ruby: 95,
|
|
107
|
+
rt: 96,
|
|
108
|
+
rp: 97,
|
|
109
|
+
dd: 98,
|
|
110
|
+
dt: 99,
|
|
111
|
+
dl: 101,
|
|
112
|
+
address: 100,
|
|
113
|
+
figure: 102,
|
|
114
|
+
object: 103,
|
|
115
|
+
main: 104,
|
|
116
|
+
header: 105,
|
|
117
|
+
figcaption: 106,
|
|
118
|
+
caption: 107
|
|
232
119
|
};
|
|
233
|
-
const MARKDOWN_STRONG = "**";
|
|
234
|
-
const MARKDOWN_EMPHASIS = "_";
|
|
235
|
-
const MARKDOWN_STRIKETHROUGH = "~~";
|
|
236
|
-
const MARKDOWN_CODE_BLOCK = "```";
|
|
237
|
-
const MARKDOWN_INLINE_CODE = "`";
|
|
238
|
-
const MARKDOWN_HORIZONTAL_RULE = "---";
|
|
239
120
|
const NO_SPACING = [0, 0];
|
|
240
121
|
const DEFAULT_BLOCK_SPACING = [2, 2];
|
|
241
122
|
const BLOCKQUOTE_SPACING = [1, 1];
|
|
242
123
|
const LIST_ITEM_SPACING = [1, 0];
|
|
243
124
|
const TABLE_ROW_SPACING = [0, 1];
|
|
244
|
-
|
|
125
|
+
//#endregion
|
|
126
|
+
export { NO_SPACING as a, LIST_ITEM_SPACING as i, DEFAULT_BLOCK_SPACING as n, TABLE_ROW_SPACING as o, HTML_ENTITIES as r, TagIdMap as s, BLOCKQUOTE_SPACING as t };
|
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
import { t as createPlugin } from "./plugin.mjs";
|
|
2
|
+
//#region src/libs/query-selector.ts
|
|
3
|
+
/**
|
|
4
|
+
* Creates a tag selector matcher (e.g., 'div', 'p', 'h1')
|
|
5
|
+
*/
|
|
2
6
|
function createTagSelector(tagName) {
|
|
3
7
|
return {
|
|
4
8
|
matches: (element) => element.name === tagName,
|
|
5
9
|
toString: () => tagName
|
|
6
10
|
};
|
|
7
11
|
}
|
|
12
|
+
/**
|
|
13
|
+
* Creates an ID selector matcher (e.g., '#main', '#content')
|
|
14
|
+
*/
|
|
8
15
|
function createIdSelector(selector) {
|
|
9
16
|
const id = selector.slice(1);
|
|
10
17
|
return {
|
|
@@ -12,6 +19,9 @@ function createIdSelector(selector) {
|
|
|
12
19
|
toString: () => `#${id}`
|
|
13
20
|
};
|
|
14
21
|
}
|
|
22
|
+
/**
|
|
23
|
+
* Creates a class selector matcher (e.g., '.container', '.header')
|
|
24
|
+
*/
|
|
15
25
|
function createClassSelector(selector) {
|
|
16
26
|
const className = selector.slice(1);
|
|
17
27
|
return {
|
|
@@ -22,8 +32,12 @@ function createClassSelector(selector) {
|
|
|
22
32
|
toString: () => `.${className}`
|
|
23
33
|
};
|
|
24
34
|
}
|
|
35
|
+
const ATTR_SELECTOR_RE = /\[([^\]=~|^$*]+)(?:([=~|^$*]+)["']?([^"'\]]+)["']?)?\]/;
|
|
36
|
+
/**
|
|
37
|
+
* Creates an attribute selector matcher (e.g., '[data-id]', '[href="https://example.com"]')
|
|
38
|
+
*/
|
|
25
39
|
function createAttributeSelector(selector) {
|
|
26
|
-
const match = selector.match(
|
|
40
|
+
const match = selector.match(ATTR_SELECTOR_RE);
|
|
27
41
|
const attrName = match ? match[1] : selector.slice(1, -1);
|
|
28
42
|
const operator = match?.[2];
|
|
29
43
|
const attrValue = match?.[3];
|
|
@@ -48,12 +62,18 @@ function createAttributeSelector(selector) {
|
|
|
48
62
|
}
|
|
49
63
|
};
|
|
50
64
|
}
|
|
65
|
+
/**
|
|
66
|
+
* Creates a compound selector that combines multiple selectors (e.g., 'div.container', 'h1#title')
|
|
67
|
+
*/
|
|
51
68
|
function createCompoundSelector(selectors) {
|
|
52
69
|
return {
|
|
53
70
|
matches: (element) => selectors.every((selector) => selector.matches(element)),
|
|
54
71
|
toString: () => selectors.map((s) => s.toString()).join("")
|
|
55
72
|
};
|
|
56
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Parses a CSS selector into a matcher
|
|
76
|
+
*/
|
|
57
77
|
function parseSelector(selector) {
|
|
58
78
|
selector = selector.trim();
|
|
59
79
|
if (!selector) throw new Error("Empty selector");
|
|
@@ -80,6 +100,8 @@ function parseSelector(selector) {
|
|
|
80
100
|
if (selectorParts.length === 1) return selectorParts[0];
|
|
81
101
|
return createCompoundSelector(selectorParts);
|
|
82
102
|
}
|
|
103
|
+
//#endregion
|
|
104
|
+
//#region src/plugins/extraction.ts
|
|
83
105
|
function extractionPlugin(selectors) {
|
|
84
106
|
const matcherCallbacks = Object.entries(selectors).map(([selector, callback]) => ({
|
|
85
107
|
matcher: parseSelector(selector),
|
|
@@ -116,4 +138,5 @@ function extractionPlugin(selectors) {
|
|
|
116
138
|
}
|
|
117
139
|
});
|
|
118
140
|
}
|
|
141
|
+
//#endregion
|
|
119
142
|
export { parseSelector as n, extractionPlugin as t };
|