@mdream/js 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/const.mjs +0 -1
- package/dist/_chunks/index.d.mts +0 -5
- package/dist/_chunks/minimal.d.mts +0 -3
- package/dist/_chunks/parse.mjs +0 -50
- package/dist/_chunks/plugins.mjs +0 -128
- package/dist/_chunks/resolve-plugins.mjs +0 -44
- package/dist/_chunks/src.mjs +3 -20
- package/dist/_chunks/types.d.mts +0 -4
- package/dist/cli.mjs +0 -6
- package/dist/index.mjs +0 -2
- package/dist/llms-txt.d.mts +0 -2
- package/dist/llms-txt.mjs +0 -39
- package/dist/negotiate.d.mts +0 -2
- package/dist/negotiate.mjs +0 -18
- package/dist/parse.d.mts +0 -3
- package/dist/parse.mjs +0 -1
- package/dist/plugins.d.mts +0 -10
- package/dist/plugins.mjs +0 -1
- package/dist/preset/minimal.mjs +0 -6
- package/dist/splitter.d.mts +0 -3
- package/dist/splitter.mjs +0 -20
- package/package.json +1 -1
package/dist/_chunks/const.mjs
CHANGED
|
@@ -124,5 +124,4 @@ const DEFAULT_BLOCK_SPACING = [2, 2];
|
|
|
124
124
|
const BLOCKQUOTE_SPACING = [1, 1];
|
|
125
125
|
const LIST_ITEM_SPACING = [1, 0];
|
|
126
126
|
const TABLE_ROW_SPACING = [0, 1];
|
|
127
|
-
//#endregion
|
|
128
127
|
export { NO_SPACING as a, TABLE_ROW_SPACING as c, TAG_H3 as d, TAG_H4 as f, TagIdMap as g, TEXT_NODE as h, LIST_ITEM_SPACING as i, TAG_H1 as l, TAG_H6 as m, DEFAULT_BLOCK_SPACING as n, NodeEventEnter as o, TAG_H5 as p, ELEMENT_NODE as r, NodeEventExit as s, BLOCKQUOTE_SPACING as t, TAG_H2 as u };
|
package/dist/_chunks/index.d.mts
CHANGED
|
@@ -1,14 +1,9 @@
|
|
|
1
1
|
import { _ as TransformPlugin, c as MdreamOptions } from "./types.mjs";
|
|
2
|
-
|
|
3
|
-
//#region src/pluggable/plugin.d.ts
|
|
4
2
|
/**
|
|
5
3
|
* Create a plugin with type-safe hook definitions.
|
|
6
4
|
* All TransformPlugin fields are optional, so this is a typed identity function.
|
|
7
5
|
*/
|
|
8
6
|
declare function createPlugin(plugin: TransformPlugin): TransformPlugin;
|
|
9
|
-
//#endregion
|
|
10
|
-
//#region src/index.d.ts
|
|
11
7
|
declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
12
8
|
declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
13
|
-
//#endregion
|
|
14
9
|
export { streamHtmlToMarkdown as n, createPlugin as r, htmlToMarkdown as t };
|
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
import { i as EngineOptions } from "./types.mjs";
|
|
2
|
-
|
|
3
|
-
//#region src/preset/minimal.d.ts
|
|
4
2
|
/**
|
|
5
3
|
* Creates a configurable minimal preset with advanced options.
|
|
6
4
|
* Returns declarative plugin config that works with both JS and Rust engines.
|
|
7
5
|
*/
|
|
8
6
|
declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;
|
|
9
|
-
//#endregion
|
|
10
7
|
export { withMinimalPreset as t };
|
package/dist/_chunks/parse.mjs
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { a as NO_SPACING, c as TABLE_ROW_SPACING, g as TagIdMap, i as LIST_ITEM_SPACING, t as BLOCKQUOTE_SPACING } from "./const.mjs";
|
|
2
|
-
//#region src/tags.ts
|
|
3
2
|
function resolveUrl(url, origin) {
|
|
4
3
|
if (!url) return url;
|
|
5
4
|
if (url.startsWith("//")) return `https:${url}`;
|
|
@@ -560,11 +559,6 @@ const tagHandlers = {
|
|
|
560
559
|
isInline: true
|
|
561
560
|
}
|
|
562
561
|
};
|
|
563
|
-
/**
|
|
564
|
-
* Build a map of tag name → TagHandler from declarative tagOverrides config.
|
|
565
|
-
* For alias (string value): clone the handler for the aliased tag.
|
|
566
|
-
* For override object: overlay fields onto the base handler (if tag is known).
|
|
567
|
-
*/
|
|
568
562
|
function buildTagOverrideHandlers(overrides) {
|
|
569
563
|
const result = /* @__PURE__ */ new Map();
|
|
570
564
|
for (const tagName in overrides) {
|
|
@@ -597,8 +591,6 @@ function buildTagOverrideHandlers(overrides) {
|
|
|
597
591
|
}
|
|
598
592
|
return result;
|
|
599
593
|
}
|
|
600
|
-
//#endregion
|
|
601
|
-
//#region src/entities.ts
|
|
602
594
|
const HTML_ENTITIES = {
|
|
603
595
|
"á": "á",
|
|
604
596
|
"Á": "Á",
|
|
@@ -846,11 +838,6 @@ const HTML_ENTITIES = {
|
|
|
846
838
|
"ζ": "ζ",
|
|
847
839
|
"Ζ": "Ζ"
|
|
848
840
|
};
|
|
849
|
-
//#endregion
|
|
850
|
-
//#region src/utils.ts
|
|
851
|
-
/**
|
|
852
|
-
* Decode HTML entities - single pass with O(1) named entity lookup
|
|
853
|
-
*/
|
|
854
841
|
function decodeHTMLEntities(text) {
|
|
855
842
|
let result = "";
|
|
856
843
|
let i = 0;
|
|
@@ -901,8 +888,6 @@ function traverseUpToFirstBlockNode(node) {
|
|
|
901
888
|
}
|
|
902
889
|
return parentsToIncrement;
|
|
903
890
|
}
|
|
904
|
-
//#endregion
|
|
905
|
-
//#region src/parse.ts
|
|
906
891
|
const LT_CHAR = 60;
|
|
907
892
|
const GT_CHAR = 62;
|
|
908
893
|
const SLASH_CHAR = 47;
|
|
@@ -925,16 +910,9 @@ const EMPTY_ATTRIBUTES = Object.freeze({});
|
|
|
925
910
|
function copyDepthMap(depthMap) {
|
|
926
911
|
return new Uint8Array(depthMap);
|
|
927
912
|
}
|
|
928
|
-
/**
|
|
929
|
-
* Fast whitespace check using direct character code comparison
|
|
930
|
-
*/
|
|
931
913
|
function isWhitespace(charCode) {
|
|
932
914
|
return charCode === SPACE_CHAR || charCode === TAB_CHAR || charCode === NEWLINE_CHAR || charCode === CARRIAGE_RETURN_CHAR;
|
|
933
915
|
}
|
|
934
|
-
/**
|
|
935
|
-
* Pure HTML parser that emits DOM events
|
|
936
|
-
* Completely decoupled from markdown generation
|
|
937
|
-
*/
|
|
938
916
|
function parseHtml(html, options = {}) {
|
|
939
917
|
const events = [];
|
|
940
918
|
return {
|
|
@@ -948,15 +926,9 @@ function parseHtml(html, options = {}) {
|
|
|
948
926
|
})
|
|
949
927
|
};
|
|
950
928
|
}
|
|
951
|
-
/**
|
|
952
|
-
* Streaming HTML parser - calls onEvent for each DOM event
|
|
953
|
-
*/
|
|
954
929
|
function parseHtmlStream(html, state, onEvent) {
|
|
955
930
|
return parseHtmlInternal(html, state, onEvent);
|
|
956
931
|
}
|
|
957
|
-
/**
|
|
958
|
-
* Internal parsing function - extracted from original parseHTML
|
|
959
|
-
*/
|
|
960
932
|
function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
961
933
|
let textBuffer = "";
|
|
962
934
|
state.depthMap ??= new Uint8Array(108);
|
|
@@ -1098,9 +1070,6 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1098
1070
|
}
|
|
1099
1071
|
return textBuffer;
|
|
1100
1072
|
}
|
|
1101
|
-
/**
|
|
1102
|
-
* Process accumulated text buffer and create text node event
|
|
1103
|
-
*/
|
|
1104
1073
|
function processTextBuffer(textBuffer, state, handleEvent) {
|
|
1105
1074
|
const containsNonWhitespace = state.textBufferContainsNonWhitespace;
|
|
1106
1075
|
const containsWhitespace = state.textBufferContainsWhitespace;
|
|
@@ -1139,9 +1108,6 @@ function processTextBuffer(textBuffer, state, handleEvent) {
|
|
|
1139
1108
|
});
|
|
1140
1109
|
state.lastTextNode = textNode;
|
|
1141
1110
|
}
|
|
1142
|
-
/**
|
|
1143
|
-
* Process HTML closing tag
|
|
1144
|
-
*/
|
|
1145
1111
|
function processClosingTag(htmlChunk, position, state, handleEvent) {
|
|
1146
1112
|
let i = position + 2;
|
|
1147
1113
|
const tagNameStart = i;
|
|
@@ -1182,9 +1148,6 @@ function processClosingTag(htmlChunk, position, state, handleEvent) {
|
|
|
1182
1148
|
remainingText: ""
|
|
1183
1149
|
};
|
|
1184
1150
|
}
|
|
1185
|
-
/**
|
|
1186
|
-
* Close a node and emit exit event
|
|
1187
|
-
*/
|
|
1188
1151
|
function closeNode(node, state, handleEvent) {
|
|
1189
1152
|
if (!node) return;
|
|
1190
1153
|
if (node.tagId === 26 && !node.childTextNodeIndex) {
|
|
@@ -1220,9 +1183,6 @@ function closeNode(node, state, handleEvent) {
|
|
|
1220
1183
|
state.hasEncodedHtmlEntity = false;
|
|
1221
1184
|
state.justClosedTag = true;
|
|
1222
1185
|
}
|
|
1223
|
-
/**
|
|
1224
|
-
* Process HTML comment or doctype
|
|
1225
|
-
*/
|
|
1226
1186
|
function processCommentOrDoctype(htmlChunk, position) {
|
|
1227
1187
|
let i = position;
|
|
1228
1188
|
const chunkLength = htmlChunk.length;
|
|
@@ -1264,9 +1224,6 @@ function processCommentOrDoctype(htmlChunk, position) {
|
|
|
1264
1224
|
};
|
|
1265
1225
|
}
|
|
1266
1226
|
}
|
|
1267
|
-
/**
|
|
1268
|
-
* Process HTML opening tag
|
|
1269
|
-
*/
|
|
1270
1227
|
function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
|
|
1271
1228
|
if (state.currentNode?.tagHandler?.isNonNesting) closeNode(state.currentNode, state, handleEvent);
|
|
1272
1229
|
const tagHandler = state.tagOverrideHandlers?.get(tagName) ?? tagHandlers[tagId];
|
|
@@ -1320,9 +1277,6 @@ function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
|
|
|
1320
1277
|
selfClosing: result.selfClosing
|
|
1321
1278
|
};
|
|
1322
1279
|
}
|
|
1323
|
-
/**
|
|
1324
|
-
* Extract and process HTML tag attributes
|
|
1325
|
-
*/
|
|
1326
1280
|
function processTagAttributes(htmlChunk, position, tagHandler) {
|
|
1327
1281
|
let i = position;
|
|
1328
1282
|
const chunkLength = htmlChunk.length;
|
|
@@ -1370,9 +1324,6 @@ function processTagAttributes(htmlChunk, position, tagHandler) {
|
|
|
1370
1324
|
attrBuffer: htmlChunk.substring(attrStartPos, i)
|
|
1371
1325
|
};
|
|
1372
1326
|
}
|
|
1373
|
-
/**
|
|
1374
|
-
* Parse HTML attributes string into key-value object
|
|
1375
|
-
*/
|
|
1376
1327
|
function parseAttributes(attrStr) {
|
|
1377
1328
|
if (!attrStr) return EMPTY_ATTRIBUTES;
|
|
1378
1329
|
const result = {};
|
|
@@ -1457,5 +1408,4 @@ function parseAttributes(attrStr) {
|
|
|
1457
1408
|
}
|
|
1458
1409
|
return result;
|
|
1459
1410
|
}
|
|
1460
|
-
//#endregion
|
|
1461
1411
|
export { buildTagOverrideHandlers as i, parseHtml as n, parseHtmlStream as r, parseAttributes as t };
|
package/dist/_chunks/plugins.mjs
CHANGED
|
@@ -1,26 +1,13 @@
|
|
|
1
1
|
import { g as TagIdMap } from "./const.mjs";
|
|
2
|
-
//#region src/pluggable/plugin.ts
|
|
3
|
-
/**
|
|
4
|
-
* Create a plugin with type-safe hook definitions.
|
|
5
|
-
* All TransformPlugin fields are optional, so this is a typed identity function.
|
|
6
|
-
*/
|
|
7
2
|
function createPlugin(plugin) {
|
|
8
3
|
return plugin;
|
|
9
4
|
}
|
|
10
|
-
//#endregion
|
|
11
|
-
//#region src/libs/query-selector.ts
|
|
12
|
-
/**
|
|
13
|
-
* Creates a tag selector matcher (e.g., 'div', 'p', 'h1')
|
|
14
|
-
*/
|
|
15
5
|
function createTagSelector(tagName) {
|
|
16
6
|
return {
|
|
17
7
|
matches: (element) => element.name === tagName,
|
|
18
8
|
toString: () => tagName
|
|
19
9
|
};
|
|
20
10
|
}
|
|
21
|
-
/**
|
|
22
|
-
* Creates an ID selector matcher (e.g., '#main', '#content')
|
|
23
|
-
*/
|
|
24
11
|
function createIdSelector(selector) {
|
|
25
12
|
const id = selector.slice(1);
|
|
26
13
|
return {
|
|
@@ -28,9 +15,6 @@ function createIdSelector(selector) {
|
|
|
28
15
|
toString: () => `#${id}`
|
|
29
16
|
};
|
|
30
17
|
}
|
|
31
|
-
/**
|
|
32
|
-
* Creates a class selector matcher (e.g., '.container', '.header')
|
|
33
|
-
*/
|
|
34
18
|
function createClassSelector(selector) {
|
|
35
19
|
const className = selector.slice(1);
|
|
36
20
|
return {
|
|
@@ -41,10 +25,6 @@ function createClassSelector(selector) {
|
|
|
41
25
|
toString: () => `.${className}`
|
|
42
26
|
};
|
|
43
27
|
}
|
|
44
|
-
/**
|
|
45
|
-
* Parses attribute selectors like [attr], [attr=value], [attr^="value"]
|
|
46
|
-
* Uses a manual parser to avoid polynomial regex backtracking (CodeQL ReDoS).
|
|
47
|
-
*/
|
|
48
28
|
function parseAttributeSelector(selector) {
|
|
49
29
|
if (selector.charCodeAt(0) !== 91) return null;
|
|
50
30
|
const end = selector.indexOf("]", 1);
|
|
@@ -71,9 +51,6 @@ function parseAttributeSelector(selector) {
|
|
|
71
51
|
value
|
|
72
52
|
};
|
|
73
53
|
}
|
|
74
|
-
/**
|
|
75
|
-
* Creates an attribute selector matcher (e.g., '[data-id]', '[href="https://example.com"]')
|
|
76
|
-
*/
|
|
77
54
|
function createAttributeSelector(selector) {
|
|
78
55
|
const parsed = parseAttributeSelector(selector);
|
|
79
56
|
const attrName = parsed ? parsed.attr : selector.slice(1, -1);
|
|
@@ -101,18 +78,12 @@ function createAttributeSelector(selector) {
|
|
|
101
78
|
}
|
|
102
79
|
};
|
|
103
80
|
}
|
|
104
|
-
/**
|
|
105
|
-
* Creates a compound selector that combines multiple selectors (e.g., 'div.container', 'h1#title')
|
|
106
|
-
*/
|
|
107
81
|
function createCompoundSelector(selectors) {
|
|
108
82
|
return {
|
|
109
83
|
matches: (element) => selectors.every((selector) => selector.matches(element)),
|
|
110
84
|
toString: () => selectors.map((s) => s.toString()).join("")
|
|
111
85
|
};
|
|
112
86
|
}
|
|
113
|
-
/**
|
|
114
|
-
* Parses a CSS selector into a matcher
|
|
115
|
-
*/
|
|
116
87
|
function parseSelector(selector) {
|
|
117
88
|
selector = selector.trim();
|
|
118
89
|
if (!selector) throw new Error("Empty selector");
|
|
@@ -141,11 +112,6 @@ function parseSelector(selector) {
|
|
|
141
112
|
if (selectorParts.length === 1) return selectorParts[0];
|
|
142
113
|
return createCompoundSelector(selectorParts);
|
|
143
114
|
}
|
|
144
|
-
//#endregion
|
|
145
|
-
//#region src/plugins/extraction.ts
|
|
146
|
-
/**
|
|
147
|
-
* @deprecated Use `plugins.extraction` config for declarative extraction that works with both JS and Rust engines.
|
|
148
|
-
*/
|
|
149
115
|
function extractionPlugin(selectors) {
|
|
150
116
|
const matcherCallbacks = Object.entries(selectors).map(([selector, callback]) => ({
|
|
151
117
|
matcher: parseSelector(selector),
|
|
@@ -182,11 +148,6 @@ function extractionPlugin(selectors) {
|
|
|
182
148
|
}
|
|
183
149
|
});
|
|
184
150
|
}
|
|
185
|
-
/**
|
|
186
|
-
* Extraction collector for `plugins.extraction` config.
|
|
187
|
-
* Collects results during processing; callbacks are called post-conversion
|
|
188
|
-
* to match Rust engine behavior.
|
|
189
|
-
*/
|
|
190
151
|
function extractionCollectorPlugin(config) {
|
|
191
152
|
const matchers = Object.entries(config).map(([selector, callback]) => ({
|
|
192
153
|
selector,
|
|
@@ -243,13 +204,6 @@ function extractionCollectorPlugin(config) {
|
|
|
243
204
|
callHandlers
|
|
244
205
|
};
|
|
245
206
|
}
|
|
246
|
-
//#endregion
|
|
247
|
-
//#region src/plugins/filter.ts
|
|
248
|
-
/**
|
|
249
|
-
* Compiles a selector (string or TAG_* number) into a fast matcher.
|
|
250
|
-
* String tag names (e.g. 'form') are compiled to TAG_* ID comparisons at creation time,
|
|
251
|
-
* avoiding per-element string comparison. CSS selectors (e.g. '.class', '#id') use parseSelector.
|
|
252
|
-
*/
|
|
253
207
|
function compileSelector(selector) {
|
|
254
208
|
if (typeof selector === "number") return {
|
|
255
209
|
matches: (element) => element.tagId === selector,
|
|
@@ -262,18 +216,6 @@ function compileSelector(selector) {
|
|
|
262
216
|
};
|
|
263
217
|
return parseSelector(selector);
|
|
264
218
|
}
|
|
265
|
-
/**
|
|
266
|
-
* Plugin that filters nodes based on CSS selectors.
|
|
267
|
-
* Allows including or excluding nodes based on selectors.
|
|
268
|
-
*
|
|
269
|
-
* @example
|
|
270
|
-
* // Include only heading elements and their children
|
|
271
|
-
* withQuerySelectorPlugin({ include: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'] })
|
|
272
|
-
*
|
|
273
|
-
* @example
|
|
274
|
-
* // Exclude navigation, sidebar, and footer
|
|
275
|
-
* withQuerySelectorPlugin({ exclude: ['nav', '#sidebar', '.footer'] })
|
|
276
|
-
*/
|
|
277
219
|
function filterPlugin(options = {}) {
|
|
278
220
|
const includeSelectors = options.include?.map((selector) => compileSelector(selector)) || [];
|
|
279
221
|
const excludeSelectors = options.exclude?.map((selector) => compileSelector(selector)) || [];
|
|
@@ -312,15 +254,9 @@ function filterPlugin(options = {}) {
|
|
|
312
254
|
}
|
|
313
255
|
} });
|
|
314
256
|
}
|
|
315
|
-
//#endregion
|
|
316
|
-
//#region src/plugins/frontmatter.ts
|
|
317
257
|
const BACKSLASH_RE = /\\/g;
|
|
318
258
|
const DOUBLE_QUOTE_RE = /"/g;
|
|
319
259
|
const ESCAPED_DOUBLE_QUOTE_RE = /\\"/g;
|
|
320
|
-
/**
|
|
321
|
-
* A plugin that manages frontmatter generation from HTML head elements
|
|
322
|
-
* Extracts metadata from meta tags and title and generates YAML frontmatter
|
|
323
|
-
*/
|
|
324
260
|
function frontmatterPlugin(options = {}) {
|
|
325
261
|
const additionalFields = options.additionalFields || {};
|
|
326
262
|
const metaFields = new Set([
|
|
@@ -399,9 +335,6 @@ function frontmatterPlugin(options = {}) {
|
|
|
399
335
|
});
|
|
400
336
|
plugin.getFrontmatter = getStructuredData;
|
|
401
337
|
return plugin;
|
|
402
|
-
/**
|
|
403
|
-
* Generate YAML frontmatter string from collected metadata
|
|
404
|
-
*/
|
|
405
338
|
function generateFrontmatter() {
|
|
406
339
|
if (Object.keys(frontmatter).length === 0) return "";
|
|
407
340
|
let yamlLines = [];
|
|
@@ -421,37 +354,6 @@ function frontmatterPlugin(options = {}) {
|
|
|
421
354
|
return `---\n${yamlLines.join("\n")}\n---\n\n`;
|
|
422
355
|
}
|
|
423
356
|
}
|
|
424
|
-
//#endregion
|
|
425
|
-
//#region src/plugins/isolate-main.ts
|
|
426
|
-
/**
|
|
427
|
-
* Plugin that isolates main content using the following priority order:
|
|
428
|
-
* 1. If an explicit <main> element exists (within 5 depth levels), use its content exclusively
|
|
429
|
-
* 2. Otherwise, find content between the first header tag (h1-h6) and first footer
|
|
430
|
-
* 3. If footer is within 5 levels of nesting from the header, use it as the end boundary
|
|
431
|
-
* 4. Exclude all content before the start marker and after the end marker
|
|
432
|
-
*
|
|
433
|
-
* @example
|
|
434
|
-
* ```html
|
|
435
|
-
* <body>
|
|
436
|
-
* <nav>Navigation (excluded)</nav>
|
|
437
|
-
* <main>
|
|
438
|
-
* <h1>Main Title (included)</h1>
|
|
439
|
-
* <p>Main content (included)</p>
|
|
440
|
-
* </main>
|
|
441
|
-
* <footer>Footer (excluded)</footer>
|
|
442
|
-
* </body>
|
|
443
|
-
* ```
|
|
444
|
-
*
|
|
445
|
-
* @example
|
|
446
|
-
* ```html
|
|
447
|
-
* <body>
|
|
448
|
-
* <nav>Navigation (excluded)</nav>
|
|
449
|
-
* <h1>Main Title (included)</h1>
|
|
450
|
-
* <p>Main content (included)</p>
|
|
451
|
-
* <footer>Footer (excluded)</footer>
|
|
452
|
-
* </body>
|
|
453
|
-
* ```
|
|
454
|
-
*/
|
|
455
357
|
function isolateMainPlugin() {
|
|
456
358
|
let mainElement = null;
|
|
457
359
|
let firstHeaderElement = null;
|
|
@@ -542,11 +444,6 @@ function isolateMainPlugin() {
|
|
|
542
444
|
}
|
|
543
445
|
} });
|
|
544
446
|
}
|
|
545
|
-
//#endregion
|
|
546
|
-
//#region src/plugins/tailwind.ts
|
|
547
|
-
/**
|
|
548
|
-
* Mapping of Tailwind classes to Markdown formatting
|
|
549
|
-
*/
|
|
550
447
|
const TAILWIND_TO_MARKDOWN_MAP = {
|
|
551
448
|
"font-bold": {
|
|
552
449
|
prefix: "**",
|
|
@@ -586,9 +483,6 @@ const TAILWIND_TO_MARKDOWN_MAP = {
|
|
|
586
483
|
"fixed": { hidden: true },
|
|
587
484
|
"sticky": { hidden: true }
|
|
588
485
|
};
|
|
589
|
-
/**
|
|
590
|
-
* Extract base class name from a responsive breakpoint variant
|
|
591
|
-
*/
|
|
592
486
|
function extractBaseClass(className) {
|
|
593
487
|
for (const bp of [
|
|
594
488
|
"sm:",
|
|
@@ -605,9 +499,6 @@ function extractBaseClass(className) {
|
|
|
605
499
|
breakpoint: ""
|
|
606
500
|
};
|
|
607
501
|
}
|
|
608
|
-
/**
|
|
609
|
-
* Sort classes by breakpoint for mobile-first processing
|
|
610
|
-
*/
|
|
611
502
|
function sortByBreakpoint(classes) {
|
|
612
503
|
const breakpointOrder = {
|
|
613
504
|
"": 0,
|
|
@@ -623,9 +514,6 @@ function sortByBreakpoint(classes) {
|
|
|
623
514
|
return (breakpointOrder[aBreakpoint] || 0) - (breakpointOrder[bBreakpoint] || 0);
|
|
624
515
|
});
|
|
625
516
|
}
|
|
626
|
-
/**
|
|
627
|
-
* Group classes by their formatting type to handle overrides
|
|
628
|
-
*/
|
|
629
517
|
function groupByFormattingType(classes) {
|
|
630
518
|
const sorted = sortByBreakpoint(classes);
|
|
631
519
|
const groups = {
|
|
@@ -651,9 +539,6 @@ function groupByFormattingType(classes) {
|
|
|
651
539
|
}
|
|
652
540
|
return groups;
|
|
653
541
|
}
|
|
654
|
-
/**
|
|
655
|
-
* Fix redundant markdown delimiters without regex
|
|
656
|
-
*/
|
|
657
542
|
function fixRedundantDelimiters(content) {
|
|
658
543
|
content = content.replaceAll("****", "**");
|
|
659
544
|
content = content.replaceAll("~~~~", "~~");
|
|
@@ -663,9 +548,6 @@ function fixRedundantDelimiters(content) {
|
|
|
663
548
|
}
|
|
664
549
|
return content;
|
|
665
550
|
}
|
|
666
|
-
/**
|
|
667
|
-
* Normalizes a list of Tailwind classes by processing breakpoints and resolving conflicts
|
|
668
|
-
*/
|
|
669
551
|
function normalizeClasses(classes) {
|
|
670
552
|
const result = [];
|
|
671
553
|
const mobileClasses = classes.filter((cls) => !hasBreakpoint(cls));
|
|
@@ -674,16 +556,10 @@ function normalizeClasses(classes) {
|
|
|
674
556
|
result.push(...breakpointClasses);
|
|
675
557
|
return result;
|
|
676
558
|
}
|
|
677
|
-
/**
|
|
678
|
-
* Check if a class has a breakpoint prefix
|
|
679
|
-
*/
|
|
680
559
|
function hasBreakpoint(className) {
|
|
681
560
|
const { breakpoint } = extractBaseClass(className);
|
|
682
561
|
return breakpoint !== "";
|
|
683
562
|
}
|
|
684
|
-
/**
|
|
685
|
-
* Process Tailwind classes for an element with mobile-first approach
|
|
686
|
-
*/
|
|
687
563
|
function processTailwindClasses(classes) {
|
|
688
564
|
let prefix = "";
|
|
689
565
|
let suffix = "";
|
|
@@ -735,9 +611,6 @@ function processTailwindClasses(classes) {
|
|
|
735
611
|
hidden
|
|
736
612
|
};
|
|
737
613
|
}
|
|
738
|
-
/**
|
|
739
|
-
* Creates a plugin that adds Tailwind class processing
|
|
740
|
-
*/
|
|
741
614
|
function tailwindPlugin() {
|
|
742
615
|
return createPlugin({
|
|
743
616
|
processAttributes(node) {
|
|
@@ -787,5 +660,4 @@ function tailwindPlugin() {
|
|
|
787
660
|
}
|
|
788
661
|
});
|
|
789
662
|
}
|
|
790
|
-
//#endregion
|
|
791
663
|
export { extractionCollectorPlugin as a, filterPlugin as i, isolateMainPlugin as n, extractionPlugin as o, frontmatterPlugin as r, createPlugin as s, tailwindPlugin as t };
|
|
@@ -1,17 +1,6 @@
|
|
|
1
1
|
import { a as NO_SPACING, n as DEFAULT_BLOCK_SPACING } from "./const.mjs";
|
|
2
2
|
import { r as parseHtmlStream } from "./parse.mjs";
|
|
3
3
|
import { a as extractionCollectorPlugin, i as filterPlugin, n as isolateMainPlugin, r as frontmatterPlugin, t as tailwindPlugin } from "./plugins.mjs";
|
|
4
|
-
//#region src/plugin-processor.ts
|
|
5
|
-
/**
|
|
6
|
-
* Processes plugins for a given node event
|
|
7
|
-
* Shared logic between markdown-processor.ts and stream.ts
|
|
8
|
-
*
|
|
9
|
-
* @param event - The node event to process
|
|
10
|
-
* @param plugins - Array of plugins to apply
|
|
11
|
-
* @param state - The current runtime state
|
|
12
|
-
* @param processEvent - Callback to process the event after plugin processing
|
|
13
|
-
* @returns true if the event should be skipped, false to continue processing
|
|
14
|
-
*/
|
|
15
4
|
function processPluginsForEvent(event, plugins, state, processEvent) {
|
|
16
5
|
if (plugins?.length) {
|
|
17
6
|
if (event.node.type === 1 && event.type === 0) {
|
|
@@ -47,11 +36,6 @@ function processPluginsForEvent(event, plugins, state, processEvent) {
|
|
|
47
36
|
processEvent(event);
|
|
48
37
|
return false;
|
|
49
38
|
}
|
|
50
|
-
//#endregion
|
|
51
|
-
//#region src/markdown-processor.ts
|
|
52
|
-
/**
|
|
53
|
-
* Determines if spacing is needed between two characters
|
|
54
|
-
*/
|
|
55
39
|
function needsSpacing(lastChar, firstChar, state) {
|
|
56
40
|
if (lastChar === " " || lastChar === "\n" || lastChar === " ") return false;
|
|
57
41
|
if (firstChar === " " || firstChar === "\n" || firstChar === " ") return false;
|
|
@@ -81,9 +65,6 @@ function needsSpacing(lastChar, firstChar, state) {
|
|
|
81
65
|
if (noSpaceAfter.has(lastChar) || noSpaceBefore.has(firstChar)) return false;
|
|
82
66
|
return true;
|
|
83
67
|
}
|
|
84
|
-
/**
|
|
85
|
-
* Determines if spacing should be added before text content
|
|
86
|
-
*/
|
|
87
68
|
function shouldAddSpacingBeforeText(lastChar, lastNode, textNode) {
|
|
88
69
|
if (!lastChar || lastChar === "\n" || lastChar === " " || lastChar === "[" || lastChar === ">") return false;
|
|
89
70
|
if (lastNode?.tagHandler?.isInline) return false;
|
|
@@ -92,9 +73,6 @@ function shouldAddSpacingBeforeText(lastChar, lastNode, textNode) {
|
|
|
92
73
|
if (firstChar === "." || firstChar === "," || firstChar === "!" || firstChar === "?" || firstChar === ":" || firstChar === ";" || firstChar === "_" || firstChar === "*" || firstChar === "`" || firstChar === ")" || firstChar === "]") return false;
|
|
93
74
|
return true;
|
|
94
75
|
}
|
|
95
|
-
/**
|
|
96
|
-
* Calculate newline configuration based on tag handler spacing config
|
|
97
|
-
*/
|
|
98
76
|
function calculateNewLineConfig(node) {
|
|
99
77
|
const tagId = node.tagId;
|
|
100
78
|
const depthMap = node.depthMap;
|
|
@@ -114,9 +92,6 @@ function calculateNewLineConfig(node) {
|
|
|
114
92
|
if (node.tagHandler?.spacing) return node.tagHandler?.spacing;
|
|
115
93
|
return DEFAULT_BLOCK_SPACING;
|
|
116
94
|
}
|
|
117
|
-
/**
|
|
118
|
-
* Creates a markdown processor that consumes DOM events and generates markdown
|
|
119
|
-
*/
|
|
120
95
|
function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverrideHandlers) {
|
|
121
96
|
const state = {
|
|
122
97
|
options,
|
|
@@ -124,9 +99,6 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
124
99
|
depthMap: new Uint8Array(108)
|
|
125
100
|
};
|
|
126
101
|
let lastYieldedLength = 0;
|
|
127
|
-
/**
|
|
128
|
-
* Process a DOM event and generate markdown
|
|
129
|
-
*/
|
|
130
102
|
function processEvent(event) {
|
|
131
103
|
const { type: eventType, node } = event;
|
|
132
104
|
const lastNode = state.lastNode;
|
|
@@ -212,9 +184,6 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
212
184
|
state.lastContentCache = fragment;
|
|
213
185
|
}
|
|
214
186
|
}
|
|
215
|
-
/**
|
|
216
|
-
* Process HTML string and generate events
|
|
217
|
-
*/
|
|
218
187
|
function processHtml(html) {
|
|
219
188
|
parseHtmlStream(html, {
|
|
220
189
|
depthMap: state.depthMap,
|
|
@@ -225,17 +194,11 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
225
194
|
processPluginsForEvent(event, resolvedPlugins, state, processEvent);
|
|
226
195
|
});
|
|
227
196
|
}
|
|
228
|
-
/**
|
|
229
|
-
* Get the final markdown output
|
|
230
|
-
*/
|
|
231
197
|
function getMarkdown() {
|
|
232
198
|
const result = state.buffer.join("").trimStart();
|
|
233
199
|
state.buffer.length = 0;
|
|
234
200
|
return result.trimEnd();
|
|
235
201
|
}
|
|
236
|
-
/**
|
|
237
|
-
* Get new markdown content since the last call (for streaming)
|
|
238
|
-
*/
|
|
239
202
|
function getMarkdownChunk() {
|
|
240
203
|
const currentContent = state.buffer.join("").trimStart();
|
|
241
204
|
const newContent = currentContent.slice(lastYieldedLength);
|
|
@@ -254,8 +217,6 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
254
217
|
state
|
|
255
218
|
};
|
|
256
219
|
}
|
|
257
|
-
//#endregion
|
|
258
|
-
//#region src/resolve-plugins.ts
|
|
259
220
|
function resolveFrontmatterOpt(opt) {
|
|
260
221
|
if (typeof opt === "function") return {
|
|
261
222
|
config: {},
|
|
@@ -267,10 +228,6 @@ function resolveFrontmatterOpt(opt) {
|
|
|
267
228
|
};
|
|
268
229
|
return { config: {} };
|
|
269
230
|
}
|
|
270
|
-
/**
|
|
271
|
-
* Resolves declarative BuiltinPlugins config into a flat TransformPlugin array.
|
|
272
|
-
* Optionally appends imperative transform plugins.
|
|
273
|
-
*/
|
|
274
231
|
function resolvePlugins(options, hooks) {
|
|
275
232
|
const plugins = [];
|
|
276
233
|
let callExtractionHandlers;
|
|
@@ -302,5 +259,4 @@ function resolvePlugins(options, hooks) {
|
|
|
302
259
|
frontmatterCallback
|
|
303
260
|
};
|
|
304
261
|
}
|
|
305
|
-
//#endregion
|
|
306
262
|
export { createMarkdownProcessor as n, processPluginsForEvent as r, resolvePlugins as t };
|
package/dist/_chunks/src.mjs
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import "./const.mjs";
|
|
1
2
|
import { i as buildTagOverrideHandlers, r as parseHtmlStream } from "./parse.mjs";
|
|
2
3
|
import { n as createMarkdownProcessor, r as processPluginsForEvent, t as resolvePlugins } from "./resolve-plugins.mjs";
|
|
3
|
-
|
|
4
|
+
import "./plugins.mjs";
|
|
5
|
+
import "../preset/minimal.mjs";
|
|
4
6
|
function resolveClean(clean) {
|
|
5
7
|
if (clean === true) return {
|
|
6
8
|
urls: true,
|
|
@@ -14,11 +16,6 @@ function resolveClean(clean) {
|
|
|
14
16
|
if (clean === false) return {};
|
|
15
17
|
return clean;
|
|
16
18
|
}
|
|
17
|
-
/**
|
|
18
|
-
* Try to parse a markdown link `[text](url)` starting at position `start` (the `[`).
|
|
19
|
-
* Returns { text, url, end } or null if not a link.
|
|
20
|
-
* Handles balanced parens in URLs (e.g. `javascript:void(0)`).
|
|
21
|
-
*/
|
|
22
19
|
function parseLink(md, start) {
|
|
23
20
|
const len = md.length;
|
|
24
21
|
let j = start + 1;
|
|
@@ -76,7 +73,6 @@ function slugify(text) {
|
|
|
76
73
|
if (lastWasDash) slug = slug.slice(0, -1);
|
|
77
74
|
return slug;
|
|
78
75
|
}
|
|
79
|
-
/** Strip inline markdown formatting from heading text for slug generation */
|
|
80
76
|
function stripHeadingFormatting(text) {
|
|
81
77
|
let result = "";
|
|
82
78
|
const len = text.length;
|
|
@@ -273,16 +269,6 @@ function applyClean(md, opts) {
|
|
|
273
269
|
if (opts.fragments) md = cleanFragments(md);
|
|
274
270
|
return md;
|
|
275
271
|
}
|
|
276
|
-
//#endregion
|
|
277
|
-
//#region src/stream.ts
|
|
278
|
-
/**
|
|
279
|
-
* Creates a markdown stream from an HTML stream
|
|
280
|
-
* @param htmlStream - ReadableStream of HTML content (as Uint8Array or string)
|
|
281
|
-
* @param options - Configuration options for conversion
|
|
282
|
-
* @param resolvedPlugins - Pre-resolved plugin instances
|
|
283
|
-
* @param tagOverrideHandlers - Tag override handlers from declarative config
|
|
284
|
-
* @returns An async generator yielding markdown chunks
|
|
285
|
-
*/
|
|
286
272
|
async function* streamHtmlToMarkdown$1(htmlStream, options = {}, resolvedPlugins = [], tagOverrideHandlers) {
|
|
287
273
|
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
288
274
|
const decoder = new TextDecoder();
|
|
@@ -315,8 +301,6 @@ async function* streamHtmlToMarkdown$1(htmlStream, options = {}, resolvedPlugins
|
|
|
315
301
|
reader.releaseLock();
|
|
316
302
|
}
|
|
317
303
|
}
|
|
318
|
-
//#endregion
|
|
319
|
-
//#region src/index.ts
|
|
320
304
|
function resolveHooks(options) {
|
|
321
305
|
return options.hooks?.length ? options.hooks : void 0;
|
|
322
306
|
}
|
|
@@ -340,5 +324,4 @@ function streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
|
340
324
|
const { plugins } = resolvePlugins(options, resolveHooks(options));
|
|
341
325
|
return streamHtmlToMarkdown$1(htmlStream, options, plugins, options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0);
|
|
342
326
|
}
|
|
343
|
-
//#endregion
|
|
344
327
|
export { streamHtmlToMarkdown as n, htmlToMarkdown as t };
|
package/dist/_chunks/types.d.mts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
//#region src/const.d.ts
|
|
2
1
|
declare const TAG_H1 = 7;
|
|
3
2
|
declare const TAG_H2 = 8;
|
|
4
3
|
declare const TAG_H3 = 9;
|
|
@@ -9,8 +8,6 @@ declare const ELEMENT_NODE = 1;
|
|
|
9
8
|
declare const TEXT_NODE = 2;
|
|
10
9
|
declare const NodeEventEnter$1 = 0;
|
|
11
10
|
declare const NodeEventExit$1 = 1;
|
|
12
|
-
//#endregion
|
|
13
|
-
//#region src/types.d.ts
|
|
14
11
|
/**
|
|
15
12
|
* Imperative hook-based transform plugins. **JavaScript engine only.**
|
|
16
13
|
* When transforms are provided, the JS engine is used regardless of engine selection.
|
|
@@ -386,5 +383,4 @@ interface SplitterOptions extends EngineOptions {
|
|
|
386
383
|
*/
|
|
387
384
|
keepSeparator?: boolean;
|
|
388
385
|
}
|
|
389
|
-
//#endregion
|
|
390
386
|
export { TAG_H3 as C, TEXT_NODE as D, TAG_H6 as E, TAG_H2 as S, TAG_H5 as T, TransformPlugin as _, ExtractedElement as a, NodeEventExit$1 as b, MdreamOptions as c, NodeEvent as d, PluginContext as f, TextNode as g, TagOverride as h, EngineOptions as i, MdreamRuntimeState as l, TagHandler as m, CleanOptions as n, FrontmatterConfig as o, SplitterOptions as p, ElementNode as r, MarkdownChunk as s, BuiltinPlugins as t, Node as u, ELEMENT_NODE as v, TAG_H4 as w, TAG_H1 as x, NodeEventEnter$1 as y };
|
package/dist/cli.mjs
CHANGED
|
@@ -1,15 +1,10 @@
|
|
|
1
1
|
import { n as streamHtmlToMarkdown } from "./_chunks/src.mjs";
|
|
2
|
-
import "./_chunks/const.mjs";
|
|
3
|
-
import "./_chunks/parse.mjs";
|
|
4
|
-
import "./_chunks/resolve-plugins.mjs";
|
|
5
|
-
import "./_chunks/plugins.mjs";
|
|
6
2
|
import { withMinimalPreset } from "./preset/minimal.mjs";
|
|
7
3
|
import { readFileSync } from "node:fs";
|
|
8
4
|
import { Readable } from "node:stream";
|
|
9
5
|
import { fileURLToPath } from "node:url";
|
|
10
6
|
import { cac } from "cac";
|
|
11
7
|
import { dirname, join } from "pathe";
|
|
12
|
-
//#region src/cli.ts
|
|
13
8
|
async function streamingConvert(options = {}) {
|
|
14
9
|
let conversionOptions = { origin: options.origin };
|
|
15
10
|
if (options.preset === "minimal") conversionOptions = withMinimalPreset(conversionOptions);
|
|
@@ -23,5 +18,4 @@ cli.command("[options]", "Convert HTML from stdin to Markdown on stdout (JS engi
|
|
|
23
18
|
await streamingConvert(opts);
|
|
24
19
|
});
|
|
25
20
|
cli.help().version(packageJson.version).parse();
|
|
26
|
-
//#endregion
|
|
27
21
|
export {};
|
package/dist/index.mjs
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import { n as streamHtmlToMarkdown, t as htmlToMarkdown } from "./_chunks/src.mjs";
|
|
2
2
|
import { d as TAG_H3, f as TAG_H4, h as TEXT_NODE, l as TAG_H1, m as TAG_H6, o as NodeEventEnter, p as TAG_H5, r as ELEMENT_NODE, s as NodeEventExit, u as TAG_H2 } from "./_chunks/const.mjs";
|
|
3
|
-
import "./_chunks/parse.mjs";
|
|
4
|
-
import "./_chunks/resolve-plugins.mjs";
|
|
5
3
|
import { s as createPlugin } from "./_chunks/plugins.mjs";
|
|
6
4
|
import { withMinimalPreset } from "./preset/minimal.mjs";
|
|
7
5
|
export { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, createPlugin, htmlToMarkdown, streamHtmlToMarkdown, withMinimalPreset };
|
package/dist/llms-txt.d.mts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
//#region src/llms-txt.d.ts
|
|
2
1
|
/**
|
|
3
2
|
* Link in llms.txt section
|
|
4
3
|
*/
|
|
@@ -85,5 +84,4 @@ interface CreateLlmsTxtStreamOptions extends Omit<LlmsTxtArtifactsOptions, 'file
|
|
|
85
84
|
* never keeping full content in memory. Creates outputDir recursively if needed.
|
|
86
85
|
*/
|
|
87
86
|
declare function createLlmsTxtStream(options: CreateLlmsTxtStreamOptions): WritableStream<ProcessedFile>;
|
|
88
|
-
//#endregion
|
|
89
87
|
export { CreateLlmsTxtStreamOptions, LlmsTxtArtifactsOptions, LlmsTxtArtifactsResult, LlmsTxtLink, LlmsTxtSection, ProcessedFile, createLlmsTxtStream, generateLlmsTxtArtifacts };
|
package/dist/llms-txt.mjs
CHANGED
|
@@ -1,13 +1,9 @@
|
|
|
1
1
|
import { join, relative } from "pathe";
|
|
2
2
|
import { mkdir, open } from "node:fs/promises";
|
|
3
|
-
//#region src/llms-txt.ts
|
|
4
3
|
const FRONTMATTER_RE = /^---\n([\s\S]*?)\n---\n([\s\S]*)$/;
|
|
5
4
|
const ANCHOR_INVALID_CHARS_RE = /[^a-z0-9]/g;
|
|
6
5
|
const LEADING_SLASH_RE = /^\//;
|
|
7
6
|
const TRAILING_SLASH_RE = /\/$/;
|
|
8
|
-
/**
|
|
9
|
-
* Generate llms.txt content
|
|
10
|
-
*/
|
|
11
7
|
function generateLlmsTxtContent(files, options) {
|
|
12
8
|
const { siteName = "Site", description, origin = "", sections, notes } = options;
|
|
13
9
|
let content = `# ${siteName}\n\n`;
|
|
@@ -31,9 +27,6 @@ function generateLlmsTxtContent(files, options) {
|
|
|
31
27
|
if (notes) content += `\n${formatNotes(notes)}`;
|
|
32
28
|
return content;
|
|
33
29
|
}
|
|
34
|
-
/**
|
|
35
|
-
* Parse frontmatter from markdown content
|
|
36
|
-
*/
|
|
37
30
|
function parseFrontmatter(content) {
|
|
38
31
|
const match = content.match(FRONTMATTER_RE);
|
|
39
32
|
if (!match) return {
|
|
@@ -56,17 +49,11 @@ function parseFrontmatter(content) {
|
|
|
56
49
|
body
|
|
57
50
|
};
|
|
58
51
|
}
|
|
59
|
-
/**
|
|
60
|
-
* Serialize frontmatter object to YAML-like format
|
|
61
|
-
*/
|
|
62
52
|
function serializeFrontmatter(data) {
|
|
63
53
|
const lines = [];
|
|
64
54
|
for (const [key, value] of Object.entries(data)) if (value !== void 0 && value !== null) lines.push(`${key}: ${String(value)}`);
|
|
65
55
|
return lines.join("\n");
|
|
66
56
|
}
|
|
67
|
-
/**
|
|
68
|
-
* Generate llms-full.txt content with complete page content
|
|
69
|
-
*/
|
|
70
57
|
function generateLlmsFullTxtContent(files, options) {
|
|
71
58
|
const { siteName = "Site", description, origin = "", sections, notes } = options;
|
|
72
59
|
let content = `# ${siteName}\n\n`;
|
|
@@ -107,9 +94,6 @@ function generateLlmsFullTxtContent(files, options) {
|
|
|
107
94
|
if (notes) content += `\n${formatNotes(notes)}`;
|
|
108
95
|
return content;
|
|
109
96
|
}
|
|
110
|
-
/**
|
|
111
|
-
* Generate individual markdown files structure
|
|
112
|
-
*/
|
|
113
97
|
function generateMarkdownFilesContent(files) {
|
|
114
98
|
const markdownFiles = [];
|
|
115
99
|
for (const file of files) {
|
|
@@ -121,9 +105,6 @@ function generateMarkdownFilesContent(files) {
|
|
|
121
105
|
}
|
|
122
106
|
return markdownFiles;
|
|
123
107
|
}
|
|
124
|
-
/**
|
|
125
|
-
* Main function to generate llms.txt artifacts from pre-processed files
|
|
126
|
-
*/
|
|
127
108
|
async function generateLlmsTxtArtifacts(options) {
|
|
128
109
|
const files = options.files;
|
|
129
110
|
const llmsTxt = generateLlmsTxtContent(files, options);
|
|
@@ -138,9 +119,6 @@ async function generateLlmsTxtArtifacts(options) {
|
|
|
138
119
|
processedFiles: files
|
|
139
120
|
};
|
|
140
121
|
}
|
|
141
|
-
/**
|
|
142
|
-
* Format a section with title, description, and links
|
|
143
|
-
*/
|
|
144
122
|
function formatSection(section) {
|
|
145
123
|
let content = `## ${section.title}\n\n`;
|
|
146
124
|
if (section.description) {
|
|
@@ -156,28 +134,18 @@ function formatSection(section) {
|
|
|
156
134
|
}
|
|
157
135
|
return content;
|
|
158
136
|
}
|
|
159
|
-
/**
|
|
160
|
-
* Format notes section
|
|
161
|
-
*/
|
|
162
137
|
function formatNotes(notes) {
|
|
163
138
|
const noteLines = Array.isArray(notes) ? notes : [notes];
|
|
164
139
|
let content = "";
|
|
165
140
|
for (const note of noteLines) content += `${note}\n\n`;
|
|
166
141
|
return content;
|
|
167
142
|
}
|
|
168
|
-
/**
|
|
169
|
-
* Get group prefix for a URL (up to 2 segments)
|
|
170
|
-
*/
|
|
171
143
|
function getGroupPrefix(url, depth) {
|
|
172
144
|
const segments = url.split("/").filter(Boolean);
|
|
173
145
|
if (segments.length === 0) return "/";
|
|
174
146
|
if (depth === 1 || segments.length === 1) return `/${segments[0]}`;
|
|
175
147
|
return `/${segments[0]}/${segments[1]}`;
|
|
176
148
|
}
|
|
177
|
-
/**
|
|
178
|
-
* Sort pages by URL path in hierarchical order (directory tree structure)
|
|
179
|
-
* Groups by up to 2 segments, with root-level pages without nesting grouped together
|
|
180
|
-
*/
|
|
181
149
|
function sortPagesByPath(pages) {
|
|
182
150
|
const twoSegmentCount = /* @__PURE__ */ new Map();
|
|
183
151
|
for (const page of pages) {
|
|
@@ -221,12 +189,6 @@ function sortPagesByPath(pages) {
|
|
|
221
189
|
return segmentsA.length - segmentsB.length;
|
|
222
190
|
});
|
|
223
191
|
}
|
|
224
|
-
/**
|
|
225
|
-
* Create a WritableStream that generates llms.txt artifacts by streaming pages to disk.
|
|
226
|
-
*
|
|
227
|
-
* Writes llms.txt (and optionally llms-full.txt) incrementally as pages are written,
|
|
228
|
-
* never keeping full content in memory. Creates outputDir recursively if needed.
|
|
229
|
-
*/
|
|
230
192
|
function createLlmsTxtStream(options) {
|
|
231
193
|
const { siteName = "Site", description, origin = "", generateFull, outputDir = process.cwd(), sections, notes } = options;
|
|
232
194
|
let llmsTxtHandle;
|
|
@@ -343,5 +305,4 @@ function createLlmsTxtStream(options) {
|
|
|
343
305
|
}
|
|
344
306
|
});
|
|
345
307
|
}
|
|
346
|
-
//#endregion
|
|
347
308
|
export { createLlmsTxtStream, generateLlmsTxtArtifacts };
|
package/dist/negotiate.d.mts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
//#region src/negotiate.d.ts
|
|
2
1
|
interface AcceptEntry {
|
|
3
2
|
type: string;
|
|
4
3
|
q: number;
|
|
@@ -22,5 +21,4 @@ declare function parseAcceptHeader(accept: string): AcceptEntry[];
|
|
|
22
21
|
* @param secFetchDest - The Sec-Fetch-Dest header value
|
|
23
22
|
*/
|
|
24
23
|
declare function shouldServeMarkdown(acceptHeader?: string, secFetchDest?: string): boolean;
|
|
25
|
-
//#endregion
|
|
26
24
|
export { parseAcceptHeader, shouldServeMarkdown };
|
package/dist/negotiate.mjs
CHANGED
|
@@ -1,8 +1,3 @@
|
|
|
1
|
-
//#region src/negotiate.ts
|
|
2
|
-
/**
|
|
3
|
-
* Parse an HTTP Accept header into an ordered list of media types with quality values.
|
|
4
|
-
* Supports quality weights (q=0.9) and preserves original position for tie-breaking.
|
|
5
|
-
*/
|
|
6
1
|
function parseAcceptHeader(accept) {
|
|
7
2
|
if (!accept) return [];
|
|
8
3
|
const entries = [];
|
|
@@ -33,18 +28,6 @@ function parseAcceptHeader(accept) {
|
|
|
33
28
|
}
|
|
34
29
|
return entries;
|
|
35
30
|
}
|
|
36
|
-
/**
|
|
37
|
-
* Determine if a client prefers markdown over HTML using proper content negotiation.
|
|
38
|
-
*
|
|
39
|
-
* Uses Accept header quality weights and position ordering:
|
|
40
|
-
* - If text/markdown or text/plain has higher quality than text/html -> markdown
|
|
41
|
-
* - If same quality, earlier position in Accept header wins
|
|
42
|
-
* - Bare wildcard does NOT trigger markdown (prevents breaking OG crawlers)
|
|
43
|
-
* - sec-fetch-dest: document always returns false (browser navigation)
|
|
44
|
-
*
|
|
45
|
-
* @param acceptHeader - The HTTP Accept header value
|
|
46
|
-
* @param secFetchDest - The Sec-Fetch-Dest header value
|
|
47
|
-
*/
|
|
48
31
|
function shouldServeMarkdown(acceptHeader, secFetchDest) {
|
|
49
32
|
if (secFetchDest === "document") return false;
|
|
50
33
|
const accept = acceptHeader || "";
|
|
@@ -88,5 +71,4 @@ function shouldServeMarkdown(acceptHeader, secFetchDest) {
|
|
|
88
71
|
if (bestMdQ === htmlQ && bestMdPos < htmlPos) return true;
|
|
89
72
|
return false;
|
|
90
73
|
}
|
|
91
|
-
//#endregion
|
|
92
74
|
export { parseAcceptHeader, shouldServeMarkdown };
|
package/dist/parse.d.mts
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
import { _ as TransformPlugin, d as NodeEvent, m as TagHandler, r as ElementNode, u as Node } from "./_chunks/types.mjs";
|
|
2
|
-
|
|
3
|
-
//#region src/parse.d.ts
|
|
4
2
|
interface ParseOptions {
|
|
5
3
|
resolvedPlugins?: TransformPlugin[];
|
|
6
4
|
}
|
|
@@ -53,5 +51,4 @@ declare function parseHtmlStream(html: string, state: ParseState, onEvent: (even
|
|
|
53
51
|
* Parse HTML attributes string into key-value object
|
|
54
52
|
*/
|
|
55
53
|
declare function parseAttributes(attrStr: string): Record<string, string>;
|
|
56
|
-
//#endregion
|
|
57
54
|
export { ParseOptions, ParseResult, ParseState, parseAttributes, parseHtml, parseHtmlStream };
|
package/dist/parse.mjs
CHANGED
package/dist/plugins.d.mts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { _ as TransformPlugin, a as ExtractedElement$1, l as MdreamRuntimeState, r as ElementNode } from "./_chunks/types.mjs";
|
|
2
2
|
import { r as createPlugin } from "./_chunks/index.mjs";
|
|
3
|
-
//#region src/plugins/extraction.d.ts
|
|
4
3
|
interface ExtractedElement extends ElementNode {
|
|
5
4
|
textContent: string;
|
|
6
5
|
}
|
|
@@ -18,8 +17,6 @@ declare function extractionCollectorPlugin(config: Record<string, (element: Extr
|
|
|
18
17
|
getResults: () => ExtractedElement$1[];
|
|
19
18
|
callHandlers: () => void;
|
|
20
19
|
};
|
|
21
|
-
//#endregion
|
|
22
|
-
//#region src/plugins/filter.d.ts
|
|
23
20
|
/**
|
|
24
21
|
* Plugin that filters nodes based on CSS selectors.
|
|
25
22
|
* Allows including or excluding nodes based on selectors.
|
|
@@ -38,8 +35,6 @@ declare function filterPlugin(options?: {
|
|
|
38
35
|
processChildren?: boolean;
|
|
39
36
|
keepAbsolute?: boolean;
|
|
40
37
|
}): TransformPlugin;
|
|
41
|
-
//#endregion
|
|
42
|
-
//#region src/plugins/frontmatter.d.ts
|
|
43
38
|
interface FrontmatterPluginOptions {
|
|
44
39
|
/** Additional frontmatter fields to include */
|
|
45
40
|
additionalFields?: Record<string, string>;
|
|
@@ -51,8 +46,6 @@ interface FrontmatterPluginOptions {
|
|
|
51
46
|
* Extracts metadata from meta tags and title and generates YAML frontmatter
|
|
52
47
|
*/
|
|
53
48
|
declare function frontmatterPlugin(options?: FrontmatterPluginOptions): TransformPlugin;
|
|
54
|
-
//#endregion
|
|
55
|
-
//#region src/plugins/isolate-main.d.ts
|
|
56
49
|
/**
|
|
57
50
|
* Plugin that isolates main content using the following priority order:
|
|
58
51
|
* 1. If an explicit <main> element exists (within 5 depth levels), use its content exclusively
|
|
@@ -83,11 +76,8 @@ declare function frontmatterPlugin(options?: FrontmatterPluginOptions): Transfor
|
|
|
83
76
|
* ```
|
|
84
77
|
*/
|
|
85
78
|
declare function isolateMainPlugin(): TransformPlugin;
|
|
86
|
-
//#endregion
|
|
87
|
-
//#region src/plugins/tailwind.d.ts
|
|
88
79
|
/**
|
|
89
80
|
* Creates a plugin that adds Tailwind class processing
|
|
90
81
|
*/
|
|
91
82
|
declare function tailwindPlugin(): TransformPlugin;
|
|
92
|
-
//#endregion
|
|
93
83
|
export { type ExtractedElement, createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };
|
package/dist/plugins.mjs
CHANGED
|
@@ -1,3 +1,2 @@
|
|
|
1
|
-
import "./_chunks/const.mjs";
|
|
2
1
|
import { a as extractionCollectorPlugin, i as filterPlugin, n as isolateMainPlugin, o as extractionPlugin, r as frontmatterPlugin, s as createPlugin, t as tailwindPlugin } from "./_chunks/plugins.mjs";
|
|
3
2
|
export { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };
|
package/dist/preset/minimal.mjs
CHANGED
|
@@ -1,9 +1,4 @@
|
|
|
1
1
|
import "../_chunks/const.mjs";
|
|
2
|
-
//#region src/preset/minimal.ts
|
|
3
|
-
/**
|
|
4
|
-
* Creates a configurable minimal preset with advanced options.
|
|
5
|
-
* Returns declarative plugin config that works with both JS and Rust engines.
|
|
6
|
-
*/
|
|
7
2
|
function withMinimalPreset(options = {}) {
|
|
8
3
|
return {
|
|
9
4
|
clean: options.clean !== void 0 ? options.clean : true,
|
|
@@ -30,5 +25,4 @@ function withMinimalPreset(options = {}) {
|
|
|
30
25
|
}
|
|
31
26
|
};
|
|
32
27
|
}
|
|
33
|
-
//#endregion
|
|
34
28
|
export { withMinimalPreset };
|
package/dist/splitter.d.mts
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
import { p as SplitterOptions, s as MarkdownChunk } from "./_chunks/types.mjs";
|
|
2
|
-
|
|
3
|
-
//#region src/splitter.d.ts
|
|
4
2
|
/**
|
|
5
3
|
* Convert HTML to Markdown and split into chunks in single pass.
|
|
6
4
|
* Yields chunks during HTML event processing for better memory efficiency.
|
|
@@ -17,5 +15,4 @@ declare function htmlToMarkdownSplitChunksStream(html: string, options?: Splitte
|
|
|
17
15
|
* Not compatible with the Rust engine.
|
|
18
16
|
*/
|
|
19
17
|
declare function htmlToMarkdownSplitChunks(html: string, options?: SplitterOptions): MarkdownChunk[];
|
|
20
|
-
//#endregion
|
|
21
18
|
export { type MarkdownChunk, type SplitterOptions, htmlToMarkdownSplitChunks, htmlToMarkdownSplitChunksStream };
|
package/dist/splitter.mjs
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import "./_chunks/const.mjs";
|
|
2
2
|
import { r as parseHtmlStream } from "./_chunks/parse.mjs";
|
|
3
3
|
import { n as createMarkdownProcessor, r as processPluginsForEvent, t as resolvePlugins } from "./_chunks/resolve-plugins.mjs";
|
|
4
|
-
import "./_chunks/plugins.mjs";
|
|
5
|
-
//#region src/splitter.ts
|
|
6
4
|
const MARKDOWN_HEADER_LINE_RE = /^#{1,6}\s+/;
|
|
7
5
|
const NEWLINE_RE = /\n/g;
|
|
8
6
|
const DEFAULT_HEADERS_TO_SPLIT_ON = [
|
|
@@ -33,19 +31,9 @@ function getCodeLanguage(node) {
|
|
|
33
31
|
function shouldSplitOnHeader(tagId, options) {
|
|
34
32
|
return options.headersToSplitOn.includes(tagId);
|
|
35
33
|
}
|
|
36
|
-
/**
|
|
37
|
-
* Get current markdown content WITHOUT clearing buffers
|
|
38
|
-
*/
|
|
39
34
|
function getCurrentMarkdown(state) {
|
|
40
35
|
return state.buffer.join("").trimStart();
|
|
41
36
|
}
|
|
42
|
-
/**
|
|
43
|
-
* Convert HTML to Markdown and split into chunks in single pass.
|
|
44
|
-
* Yields chunks during HTML event processing for better memory efficiency.
|
|
45
|
-
*
|
|
46
|
-
* **JavaScript engine only** — uses the JS engine's internal processing pipeline.
|
|
47
|
-
* Not compatible with the Rust engine.
|
|
48
|
-
*/
|
|
49
37
|
function* htmlToMarkdownSplitChunksStream(html, options = {}) {
|
|
50
38
|
const opts = createOptions(options);
|
|
51
39
|
if (opts.chunkOverlap >= opts.chunkSize) throw new Error("chunkOverlap must be less than chunkSize");
|
|
@@ -177,13 +165,6 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
|
|
|
177
165
|
}
|
|
178
166
|
yield* flushChunk();
|
|
179
167
|
}
|
|
180
|
-
/**
|
|
181
|
-
* Convert HTML to Markdown and split into chunks in single pass.
|
|
182
|
-
* Chunks are created during HTML event processing.
|
|
183
|
-
*
|
|
184
|
-
* **JavaScript engine only** — uses the JS engine's internal processing pipeline.
|
|
185
|
-
* Not compatible with the Rust engine.
|
|
186
|
-
*/
|
|
187
168
|
function htmlToMarkdownSplitChunks(html, options = {}) {
|
|
188
169
|
const opts = createOptions(options);
|
|
189
170
|
const chunks = [];
|
|
@@ -211,5 +192,4 @@ function htmlToMarkdownSplitChunks(html, options = {}) {
|
|
|
211
192
|
}
|
|
212
193
|
return chunks;
|
|
213
194
|
}
|
|
214
|
-
//#endregion
|
|
215
195
|
export { htmlToMarkdownSplitChunks, htmlToMarkdownSplitChunksStream };
|
package/package.json
CHANGED