@mdream/js 1.7.0 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/index.d.mts +3 -4
- package/dist/_chunks/markdown-processor.mjs +905 -131
- package/dist/_chunks/minimal.d.mts +1 -2
- package/dist/_chunks/parse.mjs +212 -73
- package/dist/_chunks/stream.mjs +4 -2
- package/dist/_chunks/types.d.mts +27 -27
- package/dist/cli.d.mts +1 -1
- package/dist/cli.mjs +3 -2
- package/dist/core.d.mts +4 -5
- package/dist/core.mjs +4 -2
- package/dist/index.mjs +30 -1
- package/dist/llms-txt.d.mts +8 -9
- package/dist/negotiate.d.mts +4 -5
- package/dist/parse.d.mts +7 -8
- package/dist/plugins.d.mts +7 -7
- package/dist/splitter.d.mts +3 -3
- package/package.json +1 -1
- package/dist/_chunks/src.mjs +0 -316
package/dist/core.d.mts
CHANGED
|
@@ -1,16 +1,15 @@
|
|
|
1
1
|
import { BuiltinPlugins, MdreamOptions, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
2
|
/** Imperative plugin used by the tree-shakable converter. */
|
|
3
|
-
type Plugin = TransformPlugin;
|
|
3
|
+
export type Plugin = TransformPlugin;
|
|
4
4
|
/**
|
|
5
5
|
* Options for the tree-shakable conversion entry. Import plugin factories
|
|
6
6
|
* explicitly and pass their results as an array, matching the pre-v1 JS API.
|
|
7
7
|
* Declarative built-in plugin config and cleanup live on the package root.
|
|
8
8
|
*/
|
|
9
|
-
type CoreOptions = Omit<MdreamOptions, 'plugins' | 'clean'> & {
|
|
9
|
+
export type CoreOptions = Omit<MdreamOptions, 'plugins' | 'clean'> & {
|
|
10
10
|
plugins?: Plugin[] | Pick<BuiltinPlugins, 'tagOverrides'>;
|
|
11
11
|
};
|
|
12
12
|
/** Convert HTML without bundling the optional declarative built-in plugins. */
|
|
13
|
-
declare function htmlToMarkdown(html: string, options?: CoreOptions): string;
|
|
13
|
+
export declare function htmlToMarkdown(html: string, options?: CoreOptions): string;
|
|
14
14
|
/** Stream HTML conversion without bundling declarative built-in plugins. */
|
|
15
|
-
declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: CoreOptions): AsyncIterable<string>;
|
|
16
|
-
export { CoreOptions, Plugin, htmlToMarkdown, streamHtmlToMarkdown };
|
|
15
|
+
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: CoreOptions): AsyncIterable<string>;
|
package/dist/core.mjs
CHANGED
|
@@ -14,13 +14,15 @@ function resolveTagOverrides(options) {
|
|
|
14
14
|
function htmlToMarkdown(html, options = {}) {
|
|
15
15
|
const plugins = resolveHooks(options);
|
|
16
16
|
const tagOverrides = resolveTagOverrides(options);
|
|
17
|
-
const
|
|
17
|
+
const tagOverrideHandlers = tagOverrides ? buildTagOverrideHandlers(tagOverrides) : void 0;
|
|
18
|
+
const processor = createMarkdownProcessor(options, plugins, tagOverrideHandlers);
|
|
18
19
|
processor.processHtml(html);
|
|
19
20
|
return processor.getMarkdown();
|
|
20
21
|
}
|
|
21
22
|
function streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
22
23
|
const plugins = resolveHooks(options);
|
|
23
24
|
const tagOverrides = resolveTagOverrides(options);
|
|
24
|
-
|
|
25
|
+
const tagOverrideHandlers = tagOverrides ? buildTagOverrideHandlers(tagOverrides) : void 0;
|
|
26
|
+
return streamHtmlToMarkdown$1(htmlStream, options, plugins, tagOverrideHandlers);
|
|
25
27
|
}
|
|
26
28
|
export { htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,34 @@
|
|
|
1
1
|
import { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE } from "./_chunks/const.mjs";
|
|
2
|
-
import {
|
|
2
|
+
import { buildTagOverrideHandlers } from "./_chunks/parse.mjs";
|
|
3
|
+
import { applyClean, createMarkdownProcessor, resolveClean } from "./_chunks/markdown-processor.mjs";
|
|
3
4
|
import { createPlugin } from "./_chunks/plugins.mjs";
|
|
5
|
+
import { resolvePlugins } from "./_chunks/resolve-plugins.mjs";
|
|
6
|
+
import { streamHtmlToMarkdown as streamHtmlToMarkdown$1 } from "./_chunks/stream.mjs";
|
|
4
7
|
import { withMinimalPreset } from "./preset/minimal.mjs";
|
|
8
|
+
function resolveHooks(options) {
|
|
9
|
+
return options.hooks?.length ? options.hooks : void 0;
|
|
10
|
+
}
|
|
11
|
+
function convert(html, options, hooks) {
|
|
12
|
+
const { plugins, callExtractionHandlers, getFrontmatter, frontmatterCallback } = resolvePlugins(options, hooks);
|
|
13
|
+
const tagOverrideHandlers = options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0;
|
|
14
|
+
const processor = createMarkdownProcessor(options, plugins, tagOverrideHandlers);
|
|
15
|
+
processor.processHtml(html);
|
|
16
|
+
if (getFrontmatter && frontmatterCallback) {
|
|
17
|
+
const fm = getFrontmatter();
|
|
18
|
+
if (fm) frontmatterCallback(fm);
|
|
19
|
+
}
|
|
20
|
+
callExtractionHandlers?.();
|
|
21
|
+
return processor.getMarkdown();
|
|
22
|
+
}
|
|
23
|
+
function htmlToMarkdown(html, options = {}) {
|
|
24
|
+
const markdown = convert(html, options, resolveHooks(options));
|
|
25
|
+
if (options.clean && (options.format === void 0 || options.format === "markdown")) return applyClean(markdown, resolveClean(options.clean));
|
|
26
|
+
return markdown;
|
|
27
|
+
}
|
|
28
|
+
function streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
29
|
+
const hooks = resolveHooks(options);
|
|
30
|
+
const { plugins } = resolvePlugins(options, hooks);
|
|
31
|
+
const tagOverrideHandlers = options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0;
|
|
32
|
+
return streamHtmlToMarkdown$1(htmlStream, options, plugins, tagOverrideHandlers);
|
|
33
|
+
}
|
|
5
34
|
export { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, createPlugin, htmlToMarkdown, streamHtmlToMarkdown, withMinimalPreset };
|
package/dist/llms-txt.d.mts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Link in llms.txt section
|
|
3
3
|
*/
|
|
4
|
-
interface LlmsTxtLink {
|
|
4
|
+
export interface LlmsTxtLink {
|
|
5
5
|
/** The title of the link */
|
|
6
6
|
title: string;
|
|
7
7
|
/** The description of the link */
|
|
@@ -12,7 +12,7 @@ interface LlmsTxtLink {
|
|
|
12
12
|
/**
|
|
13
13
|
* Section in llms.txt
|
|
14
14
|
*/
|
|
15
|
-
interface LlmsTxtSection {
|
|
15
|
+
export interface LlmsTxtSection {
|
|
16
16
|
/** The title of the section */
|
|
17
17
|
title: string;
|
|
18
18
|
/** The description of the section (can be array for multiple paragraphs) */
|
|
@@ -20,7 +20,7 @@ interface LlmsTxtSection {
|
|
|
20
20
|
/** The links of the section */
|
|
21
21
|
links?: LlmsTxtLink[];
|
|
22
22
|
}
|
|
23
|
-
interface LlmsTxtArtifactsOptions {
|
|
23
|
+
export interface LlmsTxtArtifactsOptions {
|
|
24
24
|
files: ProcessedFile[];
|
|
25
25
|
siteName?: string;
|
|
26
26
|
description?: string;
|
|
@@ -33,7 +33,7 @@ interface LlmsTxtArtifactsOptions {
|
|
|
33
33
|
/** Notes to write at the end */
|
|
34
34
|
notes?: string | string[];
|
|
35
35
|
}
|
|
36
|
-
interface ProcessedFile {
|
|
36
|
+
export interface ProcessedFile {
|
|
37
37
|
filePath?: string;
|
|
38
38
|
title: string;
|
|
39
39
|
content: string;
|
|
@@ -45,7 +45,7 @@ interface ProcessedFile {
|
|
|
45
45
|
author?: string;
|
|
46
46
|
};
|
|
47
47
|
}
|
|
48
|
-
interface LlmsTxtArtifactsResult {
|
|
48
|
+
export interface LlmsTxtArtifactsResult {
|
|
49
49
|
llmsTxt: string;
|
|
50
50
|
llmsFullTxt?: string;
|
|
51
51
|
markdownFiles?: {
|
|
@@ -57,11 +57,11 @@ interface LlmsTxtArtifactsResult {
|
|
|
57
57
|
/**
|
|
58
58
|
* Main function to generate llms.txt artifacts from pre-processed files
|
|
59
59
|
*/
|
|
60
|
-
declare function generateLlmsTxtArtifacts(options: LlmsTxtArtifactsOptions): Promise<LlmsTxtArtifactsResult>;
|
|
60
|
+
export declare function generateLlmsTxtArtifacts(options: LlmsTxtArtifactsOptions): Promise<LlmsTxtArtifactsResult>;
|
|
61
61
|
/**
|
|
62
62
|
* Options for creating an llms.txt stream
|
|
63
63
|
*/
|
|
64
|
-
interface CreateLlmsTxtStreamOptions extends Omit<LlmsTxtArtifactsOptions, 'files' | 'generateMarkdown'> {
|
|
64
|
+
export interface CreateLlmsTxtStreamOptions extends Omit<LlmsTxtArtifactsOptions, 'files' | 'generateMarkdown'> {
|
|
65
65
|
/** Directory to write files to (defaults to process.cwd()) */
|
|
66
66
|
outputDir?: string;
|
|
67
67
|
/** Site name for the header (defaults to 'Site') */
|
|
@@ -83,5 +83,4 @@ interface CreateLlmsTxtStreamOptions extends Omit<LlmsTxtArtifactsOptions, 'file
|
|
|
83
83
|
* Writes llms.txt (and optionally llms-full.txt) incrementally as pages are written,
|
|
84
84
|
* never keeping full content in memory. Creates outputDir recursively if needed.
|
|
85
85
|
*/
|
|
86
|
-
declare function createLlmsTxtStream(options: CreateLlmsTxtStreamOptions): WritableStream<ProcessedFile>;
|
|
87
|
-
export { CreateLlmsTxtStreamOptions, LlmsTxtArtifactsOptions, LlmsTxtArtifactsResult, LlmsTxtLink, LlmsTxtSection, ProcessedFile, createLlmsTxtStream, generateLlmsTxtArtifacts };
|
|
86
|
+
export declare function createLlmsTxtStream(options: CreateLlmsTxtStreamOptions): WritableStream<ProcessedFile>;
|
package/dist/negotiate.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
type ContentNegotiationResult = 'markdown' | 'html' | 'not-acceptable';
|
|
1
|
+
export type ContentNegotiationResult = 'markdown' | 'html' | 'not-acceptable';
|
|
2
2
|
interface AcceptEntry {
|
|
3
3
|
type: string;
|
|
4
4
|
q: number;
|
|
@@ -8,7 +8,7 @@ interface AcceptEntry {
|
|
|
8
8
|
* Parse an HTTP Accept header into an ordered list of media types with quality values.
|
|
9
9
|
* Supports quality weights (q=0.9) and preserves original position for tie-breaking.
|
|
10
10
|
*/
|
|
11
|
-
declare function parseAcceptHeader(accept: string): AcceptEntry[];
|
|
11
|
+
export declare function parseAcceptHeader(accept: string): AcceptEntry[];
|
|
12
12
|
/**
|
|
13
13
|
* Perform RFC 7231 content negotiation for HTML vs Markdown.
|
|
14
14
|
*
|
|
@@ -26,11 +26,10 @@ declare function parseAcceptHeader(accept: string): AcceptEntry[];
|
|
|
26
26
|
* - Otherwise, compares best markdown entry vs best html-or-wildcard entry
|
|
27
27
|
* by q, then by position.
|
|
28
28
|
*/
|
|
29
|
-
declare function negotiateContent(acceptHeader?: string, secFetchDest?: string): ContentNegotiationResult;
|
|
29
|
+
export declare function negotiateContent(acceptHeader?: string, secFetchDest?: string): ContentNegotiationResult;
|
|
30
30
|
/**
|
|
31
31
|
* Determine if a client prefers markdown over HTML. Convenience wrapper over
|
|
32
32
|
* {@link negotiateContent}; treats `'not-acceptable'` the same as `'html'`
|
|
33
33
|
* (callers that want 406 semantics should use `negotiateContent` directly).
|
|
34
34
|
*/
|
|
35
|
-
declare function shouldServeMarkdown(acceptHeader?: string, secFetchDest?: string): boolean;
|
|
36
|
-
export { ContentNegotiationResult, negotiateContent, parseAcceptHeader, shouldServeMarkdown };
|
|
35
|
+
export declare function shouldServeMarkdown(acceptHeader?: string, secFetchDest?: string): boolean;
|
package/dist/parse.d.mts
CHANGED
|
@@ -12,11 +12,11 @@ import { ElementNode, Node, NodeEvent, TagHandler, TransformPlugin } from "./_ch
|
|
|
12
12
|
* trailing text was scanned persist on `state`, so `processTextBuffer` commits
|
|
13
13
|
* it as if the next tag had triggered the flush.
|
|
14
14
|
*/
|
|
15
|
-
declare function finalizeParse(leftover: string, state: ParseState, handleEvent: (event: NodeEvent) => void): void;
|
|
16
|
-
interface ParseOptions {
|
|
15
|
+
export declare function finalizeParse(leftover: string, state: ParseState, handleEvent: (event: NodeEvent) => void): void;
|
|
16
|
+
export interface ParseOptions {
|
|
17
17
|
resolvedPlugins?: TransformPlugin[];
|
|
18
18
|
}
|
|
19
|
-
interface ParseState {
|
|
19
|
+
export interface ParseState {
|
|
20
20
|
/** Map of tag names to their current nesting depth - uses TypedArray for performance */
|
|
21
21
|
depthMap: Uint16Array;
|
|
22
22
|
/** Current overall nesting depth */
|
|
@@ -71,7 +71,7 @@ interface ParseState {
|
|
|
71
71
|
/** Whether emitted text should skip Markdown-only escaping */
|
|
72
72
|
plainText?: boolean;
|
|
73
73
|
}
|
|
74
|
-
interface ParseResult {
|
|
74
|
+
export interface ParseResult {
|
|
75
75
|
events: NodeEvent[];
|
|
76
76
|
remainingHtml: string;
|
|
77
77
|
}
|
|
@@ -79,13 +79,12 @@ interface ParseResult {
|
|
|
79
79
|
* Pure HTML parser that emits DOM events
|
|
80
80
|
* Completely decoupled from markdown generation
|
|
81
81
|
*/
|
|
82
|
-
declare function parseHtml(html: string, options?: ParseOptions): ParseResult;
|
|
82
|
+
export declare function parseHtml(html: string, options?: ParseOptions): ParseResult;
|
|
83
83
|
/**
|
|
84
84
|
* Streaming HTML parser - calls onEvent for each DOM event
|
|
85
85
|
*/
|
|
86
|
-
declare function parseHtmlStream(html: string, state: ParseState, onEvent: (event: NodeEvent) => void): string;
|
|
86
|
+
export declare function parseHtmlStream(html: string, state: ParseState, onEvent: (event: NodeEvent) => void): string;
|
|
87
87
|
/**
|
|
88
88
|
* Parse HTML attributes string into key-value object
|
|
89
89
|
*/
|
|
90
|
-
declare function parseAttributes(attrStr: string): Record<string, string>;
|
|
91
|
-
export { ParseOptions, ParseResult, ParseState, finalizeParse, parseAttributes, parseHtml, parseHtmlStream };
|
|
90
|
+
export declare function parseAttributes(attrStr: string): Record<string, string>;
|
package/dist/plugins.d.mts
CHANGED
|
@@ -5,13 +5,13 @@ interface ExtractedElement extends ElementNode {
|
|
|
5
5
|
}
|
|
6
6
|
type ExtractionCallback = (element: ExtractedElement, state: MdreamRuntimeState) => void;
|
|
7
7
|
/** Extract matching elements through the composable JavaScript plugin interface. */
|
|
8
|
-
declare function extractionPlugin(selectors: Record<string, ExtractionCallback>): TransformPlugin;
|
|
8
|
+
export declare function extractionPlugin(selectors: Record<string, ExtractionCallback>): TransformPlugin;
|
|
9
9
|
/**
|
|
10
10
|
* Extraction collector for `plugins.extraction` config.
|
|
11
11
|
* Collects results during processing; callbacks are called post-conversion
|
|
12
12
|
* to match Rust engine behavior.
|
|
13
13
|
*/
|
|
14
|
-
declare function extractionCollectorPlugin(config: Record<string, (element: ExtractedElement$1) => void>): {
|
|
14
|
+
export declare function extractionCollectorPlugin(config: Record<string, (element: ExtractedElement$1) => void>): {
|
|
15
15
|
plugin: TransformPlugin;
|
|
16
16
|
getResults: () => ExtractedElement$1[];
|
|
17
17
|
callHandlers: () => void;
|
|
@@ -28,7 +28,7 @@ declare function extractionCollectorPlugin(config: Record<string, (element: Extr
|
|
|
28
28
|
* // Exclude navigation, sidebar, and footer
|
|
29
29
|
* withQuerySelectorPlugin({ exclude: ['nav', '#sidebar', '.footer'] })
|
|
30
30
|
*/
|
|
31
|
-
declare function filterPlugin(options?: {
|
|
31
|
+
export declare function filterPlugin(options?: {
|
|
32
32
|
/** CSS selectors, tag names, or TAG_* constants for elements to include (all others will be excluded) */
|
|
33
33
|
include?: (string | number)[];
|
|
34
34
|
/** CSS selectors, tag names, or TAG_* constants for elements to exclude */
|
|
@@ -47,7 +47,7 @@ interface FrontmatterPluginOptions {
|
|
|
47
47
|
* A plugin that manages frontmatter generation from HTML head elements
|
|
48
48
|
* Extracts metadata from meta tags and title and generates YAML frontmatter
|
|
49
49
|
*/
|
|
50
|
-
declare function frontmatterPlugin(options?: FrontmatterPluginOptions): TransformPlugin;
|
|
50
|
+
export declare function frontmatterPlugin(options?: FrontmatterPluginOptions): TransformPlugin;
|
|
51
51
|
/**
|
|
52
52
|
* Plugin that isolates main content using the following priority order:
|
|
53
53
|
* 1. If an explicit <main> element exists (within 5 depth levels), use its content exclusively
|
|
@@ -77,9 +77,9 @@ declare function frontmatterPlugin(options?: FrontmatterPluginOptions): Transfor
|
|
|
77
77
|
* </body>
|
|
78
78
|
* ```
|
|
79
79
|
*/
|
|
80
|
-
declare function isolateMainPlugin(): TransformPlugin;
|
|
80
|
+
export declare function isolateMainPlugin(): TransformPlugin;
|
|
81
81
|
/**
|
|
82
82
|
* Creates a plugin that adds Tailwind class processing
|
|
83
83
|
*/
|
|
84
|
-
declare function tailwindPlugin(): TransformPlugin;
|
|
85
|
-
export { type ExtractedElement, createPlugin
|
|
84
|
+
export declare function tailwindPlugin(): TransformPlugin;
|
|
85
|
+
export { type ExtractedElement, createPlugin };
|
package/dist/splitter.d.mts
CHANGED
|
@@ -6,7 +6,7 @@ import { MarkdownChunk, SplitterOptions } from "./_chunks/types.mjs";
|
|
|
6
6
|
* **JavaScript engine only** — uses the JS engine's internal processing pipeline.
|
|
7
7
|
* Not compatible with the Rust engine.
|
|
8
8
|
*/
|
|
9
|
-
declare function htmlToMarkdownSplitChunksStream(html: string, options?: SplitterOptions): Generator<MarkdownChunk, void, undefined>;
|
|
9
|
+
export declare function htmlToMarkdownSplitChunksStream(html: string, options?: SplitterOptions): Generator<MarkdownChunk, void, undefined>;
|
|
10
10
|
/**
|
|
11
11
|
* Convert HTML to Markdown and split into chunks in single pass.
|
|
12
12
|
* Chunks are created during HTML event processing.
|
|
@@ -14,5 +14,5 @@ declare function htmlToMarkdownSplitChunksStream(html: string, options?: Splitte
|
|
|
14
14
|
* **JavaScript engine only** — uses the JS engine's internal processing pipeline.
|
|
15
15
|
* Not compatible with the Rust engine.
|
|
16
16
|
*/
|
|
17
|
-
declare function htmlToMarkdownSplitChunks(html: string, options?: SplitterOptions): MarkdownChunk[];
|
|
18
|
-
export {
|
|
17
|
+
export declare function htmlToMarkdownSplitChunks(html: string, options?: SplitterOptions): MarkdownChunk[];
|
|
18
|
+
export type { MarkdownChunk, SplitterOptions };
|
package/package.json
CHANGED
package/dist/_chunks/src.mjs
DELETED
|
@@ -1,316 +0,0 @@
|
|
|
1
|
-
import "./const.mjs";
|
|
2
|
-
import { buildTagOverrideHandlers, isEmptyLinkHref } from "./parse.mjs";
|
|
3
|
-
import { createMarkdownProcessor } from "./markdown-processor.mjs";
|
|
4
|
-
import "./plugins.mjs";
|
|
5
|
-
import { resolvePlugins } from "./resolve-plugins.mjs";
|
|
6
|
-
import { streamHtmlToMarkdown as streamHtmlToMarkdown$1 } from "./stream.mjs";
|
|
7
|
-
import "../preset/minimal.mjs";
|
|
8
|
-
function resolveClean(clean) {
|
|
9
|
-
if (clean === true) return {
|
|
10
|
-
urls: true,
|
|
11
|
-
fragments: true,
|
|
12
|
-
emptyLinks: true,
|
|
13
|
-
redundantLinks: true,
|
|
14
|
-
selfLinkHeadings: true,
|
|
15
|
-
emptyImages: true,
|
|
16
|
-
emptyLinkText: true
|
|
17
|
-
};
|
|
18
|
-
if (clean === false) return {};
|
|
19
|
-
return clean;
|
|
20
|
-
}
|
|
21
|
-
function parseLink(md, start) {
|
|
22
|
-
const len = md.length;
|
|
23
|
-
let j = start + 1;
|
|
24
|
-
let depth = 1;
|
|
25
|
-
while (j < len && depth > 0) {
|
|
26
|
-
const c = md.charCodeAt(j);
|
|
27
|
-
if (c === 91) depth++;
|
|
28
|
-
else if (c === 93) depth--;
|
|
29
|
-
j++;
|
|
30
|
-
}
|
|
31
|
-
if (depth !== 0) return null;
|
|
32
|
-
const textEnd = j - 1;
|
|
33
|
-
if (j >= len || md.charCodeAt(j) !== 40) return null;
|
|
34
|
-
j++;
|
|
35
|
-
const urlStart = j;
|
|
36
|
-
let parenDepth = 1;
|
|
37
|
-
while (j < len && parenDepth > 0) {
|
|
38
|
-
const c = md.charCodeAt(j);
|
|
39
|
-
if (c === 40) parenDepth++;
|
|
40
|
-
else if (c === 41) parenDepth--;
|
|
41
|
-
j++;
|
|
42
|
-
}
|
|
43
|
-
if (parenDepth !== 0) return null;
|
|
44
|
-
const urlEnd = j - 1;
|
|
45
|
-
return {
|
|
46
|
-
text: md.slice(start + 1, textEnd),
|
|
47
|
-
url: md.slice(urlStart, urlEnd),
|
|
48
|
-
end: j
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
function slugify(text) {
|
|
52
|
-
let slug = "";
|
|
53
|
-
let lastWasDash = false;
|
|
54
|
-
for (let i = 0; i < text.length; i++) {
|
|
55
|
-
const c = text.charCodeAt(i);
|
|
56
|
-
if (c >= 97 && c <= 122) {
|
|
57
|
-
slug += text[i];
|
|
58
|
-
lastWasDash = false;
|
|
59
|
-
} else if (c >= 65 && c <= 90) {
|
|
60
|
-
slug += String.fromCharCode(c + 32);
|
|
61
|
-
lastWasDash = false;
|
|
62
|
-
} else if (c >= 48 && c <= 57) {
|
|
63
|
-
slug += text[i];
|
|
64
|
-
lastWasDash = false;
|
|
65
|
-
} else if (c === 95) {
|
|
66
|
-
slug += "_";
|
|
67
|
-
lastWasDash = false;
|
|
68
|
-
} else if (c === 32 || c === 9 || c === 45) {
|
|
69
|
-
if (!lastWasDash && slug.length > 0) {
|
|
70
|
-
slug += "-";
|
|
71
|
-
lastWasDash = true;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
if (lastWasDash) slug = slug.slice(0, -1);
|
|
76
|
-
return slug;
|
|
77
|
-
}
|
|
78
|
-
function stripHeadingFormatting(text) {
|
|
79
|
-
let result = "";
|
|
80
|
-
const len = text.length;
|
|
81
|
-
let i = 0;
|
|
82
|
-
while (i < len) {
|
|
83
|
-
const c = text.charCodeAt(i);
|
|
84
|
-
if (c === 91) {
|
|
85
|
-
const link = parseLink(text, i);
|
|
86
|
-
if (link) {
|
|
87
|
-
result += link.text;
|
|
88
|
-
i = link.end;
|
|
89
|
-
continue;
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
if (c === 42 || c === 95 || c === 96 || c === 126) {
|
|
93
|
-
i++;
|
|
94
|
-
continue;
|
|
95
|
-
}
|
|
96
|
-
result += text[i];
|
|
97
|
-
i++;
|
|
98
|
-
}
|
|
99
|
-
return result.trim();
|
|
100
|
-
}
|
|
101
|
-
function collectHeadingSlugs(md) {
|
|
102
|
-
const slugs = /* @__PURE__ */ new Set();
|
|
103
|
-
const len = md.length;
|
|
104
|
-
let i = 0;
|
|
105
|
-
while (i < len) {
|
|
106
|
-
if (i === 0 || md.charCodeAt(i - 1) === 10) {
|
|
107
|
-
let hashes = 0;
|
|
108
|
-
let j = i;
|
|
109
|
-
while (j < len && md.charCodeAt(j) === 35) {
|
|
110
|
-
hashes++;
|
|
111
|
-
j++;
|
|
112
|
-
}
|
|
113
|
-
if (hashes >= 1 && hashes <= 6 && j < len && md.charCodeAt(j) === 32) {
|
|
114
|
-
j++;
|
|
115
|
-
const lineEnd = md.indexOf("\n", j);
|
|
116
|
-
const cleaned = stripHeadingFormatting(lineEnd === -1 ? md.slice(j) : md.slice(j, lineEnd));
|
|
117
|
-
if (cleaned) slugs.add(slugify(cleaned));
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
const nl = md.indexOf("\n", i);
|
|
121
|
-
if (nl === -1) break;
|
|
122
|
-
i = nl + 1;
|
|
123
|
-
}
|
|
124
|
-
return slugs;
|
|
125
|
-
}
|
|
126
|
-
function cleanFragments(md) {
|
|
127
|
-
const slugs = collectHeadingSlugs(md);
|
|
128
|
-
const len = md.length;
|
|
129
|
-
let result = "";
|
|
130
|
-
let i = 0;
|
|
131
|
-
while (i < len) {
|
|
132
|
-
if (md.charCodeAt(i) === 91) {
|
|
133
|
-
const link = parseLink(md, i);
|
|
134
|
-
if (link && link.url.charCodeAt(0) === 35 && link.url.length > 1) {
|
|
135
|
-
const fragment = link.url.slice(1).split(" ")[0];
|
|
136
|
-
if (slugs.size > 0 ? !slugs.has(fragment) : true) {
|
|
137
|
-
result += link.text;
|
|
138
|
-
i = link.end;
|
|
139
|
-
continue;
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
result += md[i];
|
|
144
|
-
i++;
|
|
145
|
-
}
|
|
146
|
-
return result;
|
|
147
|
-
}
|
|
148
|
-
function cleanEmptyLinks(md) {
|
|
149
|
-
const len = md.length;
|
|
150
|
-
let result = "";
|
|
151
|
-
let i = 0;
|
|
152
|
-
while (i < len) {
|
|
153
|
-
if (md.charCodeAt(i) === 91) {
|
|
154
|
-
const link = parseLink(md, i);
|
|
155
|
-
if (link) {
|
|
156
|
-
const url = link.url;
|
|
157
|
-
if (isEmptyLinkHref(url)) {
|
|
158
|
-
result += link.text;
|
|
159
|
-
i = link.end;
|
|
160
|
-
continue;
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
result += md[i];
|
|
165
|
-
i++;
|
|
166
|
-
}
|
|
167
|
-
return result;
|
|
168
|
-
}
|
|
169
|
-
function isAutolinkUri(s) {
|
|
170
|
-
if (!(s.startsWith("http://") || s.startsWith("https://") || s.startsWith("ftp://") || s.startsWith("mailto:"))) return false;
|
|
171
|
-
for (let i = 0; i < s.length; i++) {
|
|
172
|
-
const c = s.charCodeAt(i);
|
|
173
|
-
if (c === 32 || c === 60 || c === 62 || c === 10 || c === 13 || c === 9) return false;
|
|
174
|
-
}
|
|
175
|
-
return true;
|
|
176
|
-
}
|
|
177
|
-
function cleanRedundantLinks(md) {
|
|
178
|
-
const len = md.length;
|
|
179
|
-
let result = "";
|
|
180
|
-
let i = 0;
|
|
181
|
-
while (i < len) {
|
|
182
|
-
const code = md.charCodeAt(i);
|
|
183
|
-
if (code === 91) {
|
|
184
|
-
const link = parseLink(md, i);
|
|
185
|
-
if (link && link.text === link.url) {
|
|
186
|
-
result += link.text;
|
|
187
|
-
i = link.end;
|
|
188
|
-
continue;
|
|
189
|
-
}
|
|
190
|
-
} else if (code === 60) {
|
|
191
|
-
const close = md.indexOf(">", i + 1);
|
|
192
|
-
if (close !== -1) {
|
|
193
|
-
const inner = md.slice(i + 1, close);
|
|
194
|
-
if (isAutolinkUri(inner)) {
|
|
195
|
-
result += inner;
|
|
196
|
-
i = close + 1;
|
|
197
|
-
continue;
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
result += md[i];
|
|
202
|
-
i++;
|
|
203
|
-
}
|
|
204
|
-
return result;
|
|
205
|
-
}
|
|
206
|
-
function cleanSelfLinkHeadings(md) {
|
|
207
|
-
const len = md.length;
|
|
208
|
-
let result = "";
|
|
209
|
-
let i = 0;
|
|
210
|
-
while (i < len) {
|
|
211
|
-
if (i === 0 || md.charCodeAt(i - 1) === 10) {
|
|
212
|
-
let hashes = 0;
|
|
213
|
-
let j = i;
|
|
214
|
-
while (j < len && md.charCodeAt(j) === 35) {
|
|
215
|
-
hashes++;
|
|
216
|
-
j++;
|
|
217
|
-
}
|
|
218
|
-
if (hashes >= 1 && hashes <= 6 && j < len && md.charCodeAt(j) === 32) {
|
|
219
|
-
j++;
|
|
220
|
-
if (j < len && md.charCodeAt(j) === 91) {
|
|
221
|
-
const link = parseLink(md, j);
|
|
222
|
-
if (link && link.url.charCodeAt(0) === 35) {
|
|
223
|
-
result += md.slice(i, j);
|
|
224
|
-
result += link.text;
|
|
225
|
-
i = link.end;
|
|
226
|
-
continue;
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
result += md[i];
|
|
232
|
-
i++;
|
|
233
|
-
}
|
|
234
|
-
return result;
|
|
235
|
-
}
|
|
236
|
-
function cleanEmptyImages(md) {
|
|
237
|
-
const len = md.length;
|
|
238
|
-
let result = "";
|
|
239
|
-
let i = 0;
|
|
240
|
-
while (i < len) {
|
|
241
|
-
if (md.charCodeAt(i) === 33 && i + 1 < len && md.charCodeAt(i + 1) === 91) {
|
|
242
|
-
let j = i + 2;
|
|
243
|
-
const altStart = j;
|
|
244
|
-
while (j < len && md.charCodeAt(j) !== 93) j++;
|
|
245
|
-
if (j < len) {
|
|
246
|
-
if (md.slice(altStart, j).trim().length === 0) {
|
|
247
|
-
j++;
|
|
248
|
-
if (j < len && md.charCodeAt(j) === 40) {
|
|
249
|
-
let parenDepth = 1;
|
|
250
|
-
j++;
|
|
251
|
-
while (j < len && parenDepth > 0) {
|
|
252
|
-
if (md.charCodeAt(j) === 40) parenDepth++;
|
|
253
|
-
else if (md.charCodeAt(j) === 41) parenDepth--;
|
|
254
|
-
j++;
|
|
255
|
-
}
|
|
256
|
-
i = j;
|
|
257
|
-
continue;
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
result += md[i];
|
|
263
|
-
i++;
|
|
264
|
-
}
|
|
265
|
-
return result;
|
|
266
|
-
}
|
|
267
|
-
function cleanEmptyLinkText(md) {
|
|
268
|
-
const len = md.length;
|
|
269
|
-
let result = "";
|
|
270
|
-
let i = 0;
|
|
271
|
-
while (i < len) {
|
|
272
|
-
if (md.charCodeAt(i) === 91) {
|
|
273
|
-
const link = parseLink(md, i);
|
|
274
|
-
if (link && link.text.trim().length === 0) {
|
|
275
|
-
i = link.end;
|
|
276
|
-
continue;
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
result += md[i];
|
|
280
|
-
i++;
|
|
281
|
-
}
|
|
282
|
-
return result;
|
|
283
|
-
}
|
|
284
|
-
function applyClean(md, opts) {
|
|
285
|
-
if (opts.emptyImages) md = cleanEmptyImages(md);
|
|
286
|
-
if (opts.emptyLinks) md = cleanEmptyLinks(md);
|
|
287
|
-
if (opts.emptyLinkText) md = cleanEmptyLinkText(md);
|
|
288
|
-
if (opts.redundantLinks) md = cleanRedundantLinks(md);
|
|
289
|
-
if (opts.selfLinkHeadings) md = cleanSelfLinkHeadings(md);
|
|
290
|
-
if (opts.fragments) md = cleanFragments(md);
|
|
291
|
-
return md;
|
|
292
|
-
}
|
|
293
|
-
function resolveHooks(options) {
|
|
294
|
-
return options.hooks?.length ? options.hooks : void 0;
|
|
295
|
-
}
|
|
296
|
-
function convert(html, options, hooks) {
|
|
297
|
-
const { plugins, callExtractionHandlers, getFrontmatter, frontmatterCallback } = resolvePlugins(options, hooks);
|
|
298
|
-
const processor = createMarkdownProcessor(options, plugins, options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0);
|
|
299
|
-
processor.processHtml(html);
|
|
300
|
-
if (getFrontmatter && frontmatterCallback) {
|
|
301
|
-
const fm = getFrontmatter();
|
|
302
|
-
if (fm) frontmatterCallback(fm);
|
|
303
|
-
}
|
|
304
|
-
callExtractionHandlers?.();
|
|
305
|
-
return processor.getMarkdown();
|
|
306
|
-
}
|
|
307
|
-
function htmlToMarkdown(html, options = {}) {
|
|
308
|
-
const markdown = convert(html, options, resolveHooks(options));
|
|
309
|
-
if (options.clean && (options.format === void 0 || options.format === "markdown")) return applyClean(markdown, resolveClean(options.clean));
|
|
310
|
-
return markdown;
|
|
311
|
-
}
|
|
312
|
-
function streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
313
|
-
const { plugins } = resolvePlugins(options, resolveHooks(options));
|
|
314
|
-
return streamHtmlToMarkdown$1(htmlStream, options, plugins, options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0);
|
|
315
|
-
}
|
|
316
|
-
export { htmlToMarkdown, streamHtmlToMarkdown };
|