page-markdown 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +186 -110
- package/README.md +106 -14
- package/dist/types/clean.d.ts +23 -0
- package/dist/types/convert.d.ts +19 -0
- package/dist/types/extract.d.ts +37 -0
- package/dist/types/facts.d.ts +14 -0
- package/dist/types/hidden.d.ts +14 -0
- package/dist/types/index.d.ts +98 -0
- package/dist/types/metadata.d.ts +18 -0
- package/dist/types/refine.d.ts +31 -0
- package/dist/types/sanitize.d.ts +26 -0
- package/dist/types/standardize.d.ts +15 -0
- package/dist/types/types.d.ts +154 -0
- package/package.json +18 -6
- package/src/clean.js +21 -9
- package/src/convert.js +78 -26
- package/src/extract.js +76 -70
- package/src/facts.js +99 -96
- package/src/hidden.js +46 -44
- package/src/index.js +21 -5
- package/src/metadata.js +79 -78
- package/src/refine.js +109 -104
- package/src/sanitize.js +196 -193
- package/src/standardize.js +41 -15
- package/src/types.js +118 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @typedef {object} PageMarkdownOptions
|
|
3
|
+
* @property {string} [url] page URL for absolute links (default: the document's URL)
|
|
4
|
+
* @property {"full"|"no_links"|"text_only"} [mode="full"]
|
|
5
|
+
* @property {"readable"|"full"|import("./extract.js").Extractor} [extract="readable"]
|
|
6
|
+
* "readable": Readability with full-page fallback; "full": the whole body;
|
|
7
|
+
* or a custom extractor receiving a copy of the standardized document.
|
|
8
|
+
* @property {number} [minExtractChars=100] below this, fall back to the full page
|
|
9
|
+
* @property {boolean} [includeFullPage=false] also return the full-page Markdown
|
|
10
|
+
* @property {boolean} [includeTitle=true] start readable output with "# <headline>"
|
|
11
|
+
* @property {boolean} [recoverLead=true] restore a dropped lead paragraph that matches the meta description
|
|
12
|
+
* @property {boolean} [pageFacts=false] append "## Page facts": short label/value pairs
|
|
13
|
+
* (sidebar versions, prices, spec rows) that the extracted article left out
|
|
14
|
+
* @property {boolean} [removeHiddenText=false] remove tag-character text,
|
|
15
|
+
* variation-selector runs and invisible operators from the Markdown (fenced
|
|
16
|
+
* code included), fullPageMarkdown and metadata; omissions count the runs
|
|
17
|
+
* removed from the Markdown
|
|
18
|
+
* @property {import("./types.js").SanitizeOptions} [sanitize] see DEFAULT_SANITIZE_OPTIONS
|
|
19
|
+
* @property {import("./types.js").ReadabilityOptions} [readability] options passed to Readability
|
|
20
|
+
*/
|
|
21
|
+
/**
|
|
22
|
+
* Converts a page (or an element of it) to Markdown.
|
|
23
|
+
* Pure with respect to the input: the live document is only read.
|
|
24
|
+
*
|
|
25
|
+
* @param {Document|Element} input
|
|
26
|
+
* @param {PageMarkdownOptions} [options]
|
|
27
|
+
* @returns {import("./types.js").PageMarkdownResult}
|
|
28
|
+
*/
|
|
29
|
+
export function pageToMarkdown(input: Document | Element, options?: PageMarkdownOptions): import("./types.js").PageMarkdownResult;
|
|
30
|
+
export const VERSION: "0.1.5";
|
|
31
|
+
export { standardizeDocument } from "./standardize.js";
|
|
32
|
+
export { collectMetadata } from "./metadata.js";
|
|
33
|
+
export * from "./types.js";
|
|
34
|
+
export type PageMarkdownOptions = {
|
|
35
|
+
/**
|
|
36
|
+
* page URL for absolute links (default: the document's URL)
|
|
37
|
+
*/
|
|
38
|
+
url?: string | undefined;
|
|
39
|
+
mode?: "full" | "no_links" | "text_only" | undefined;
|
|
40
|
+
/**
|
|
41
|
+
* "readable": Readability with full-page fallback; "full": the whole body;
|
|
42
|
+
* or a custom extractor receiving a copy of the standardized document.
|
|
43
|
+
*/
|
|
44
|
+
extract?: import("./extract.js").Extractor | "readable" | "full" | undefined;
|
|
45
|
+
/**
|
|
46
|
+
* below this, fall back to the full page
|
|
47
|
+
*/
|
|
48
|
+
minExtractChars?: number | undefined;
|
|
49
|
+
/**
|
|
50
|
+
* also return the full-page Markdown
|
|
51
|
+
*/
|
|
52
|
+
includeFullPage?: boolean | undefined;
|
|
53
|
+
/**
|
|
54
|
+
* start readable output with "# <headline>"
|
|
55
|
+
*/
|
|
56
|
+
includeTitle?: boolean | undefined;
|
|
57
|
+
/**
|
|
58
|
+
* restore a dropped lead paragraph that matches the meta description
|
|
59
|
+
*/
|
|
60
|
+
recoverLead?: boolean | undefined;
|
|
61
|
+
/**
|
|
62
|
+
* append "## Page facts": short label/value pairs
|
|
63
|
+
* (sidebar versions, prices, spec rows) that the extracted article left out
|
|
64
|
+
*/
|
|
65
|
+
pageFacts?: boolean | undefined;
|
|
66
|
+
/**
|
|
67
|
+
* remove tag-character text,
|
|
68
|
+
* variation-selector runs and invisible operators from the Markdown (fenced
|
|
69
|
+
* code included), fullPageMarkdown and metadata; omissions count the runs
|
|
70
|
+
* removed from the Markdown
|
|
71
|
+
*/
|
|
72
|
+
removeHiddenText?: boolean | undefined;
|
|
73
|
+
/**
|
|
74
|
+
* see DEFAULT_SANITIZE_OPTIONS
|
|
75
|
+
*/
|
|
76
|
+
sanitize?: import("./types.js").SanitizeOptions | undefined;
|
|
77
|
+
/**
|
|
78
|
+
* options passed to Readability
|
|
79
|
+
*/
|
|
80
|
+
readability?: {
|
|
81
|
+
debug?: boolean;
|
|
82
|
+
maxElemsToParse?: number;
|
|
83
|
+
nbTopCandidates?: number;
|
|
84
|
+
charThreshold?: number;
|
|
85
|
+
classesToPreserve?: string[];
|
|
86
|
+
keepClasses?: boolean;
|
|
87
|
+
serializer?: ((node: Node) => string) | undefined;
|
|
88
|
+
disableJSONLD?: boolean;
|
|
89
|
+
allowedVideoRegex?: RegExp;
|
|
90
|
+
} | undefined;
|
|
91
|
+
};
|
|
92
|
+
export { sanitizeToInertDocument, DEFAULT_SANITIZE_OPTIONS, isAdvertisementElement, createInertDocument } from "./sanitize.js";
|
|
93
|
+
export { extractContent, DEFAULT_MIN_EXTRACT_CHARS } from "./extract.js";
|
|
94
|
+
export { htmlToMarkdown, createTurndown, TURNDOWN_OPTIONS } from "./convert.js";
|
|
95
|
+
export { cleanMarkdown, applyMode, mapOutsideFences } from "./clean.js";
|
|
96
|
+
export { recoverLeadParagraph, chooseHeadline } from "./refine.js";
|
|
97
|
+
export { collectPageFacts, pageFactsMarkdown } from "./facts.js";
|
|
98
|
+
export { removeHiddenText, hiddenTextOmissions } from "./hidden.js";
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @param {Document} doc the original page document
|
|
3
|
+
* @param {{ url?: string }} [options]
|
|
4
|
+
*/
|
|
5
|
+
export function collectMetadata(doc: Document, options?: {
|
|
6
|
+
url?: string;
|
|
7
|
+
}): {
|
|
8
|
+
url: string;
|
|
9
|
+
canonicalUrl: string;
|
|
10
|
+
documentTitle: string;
|
|
11
|
+
title: string;
|
|
12
|
+
description: string;
|
|
13
|
+
author: string;
|
|
14
|
+
published: string;
|
|
15
|
+
modified: string;
|
|
16
|
+
siteName: string;
|
|
17
|
+
lang: string;
|
|
18
|
+
};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/** The text of serialized HTML: tags to spaces, the entities serializers write decoded. */
|
|
2
|
+
export function htmlText(html: any): string;
|
|
3
|
+
/**
|
|
4
|
+
* Many sites state their summary as both the meta description and the first
|
|
5
|
+
* paragraph. Extractors can drop that paragraph when it sits in a section
|
|
6
|
+
* without a heading (MDN). If the description matches a paragraph in the page
|
|
7
|
+
* that the extracted HTML lacks, put that paragraph first.
|
|
8
|
+
*
|
|
9
|
+
* @param {string} html extracted HTML
|
|
10
|
+
* @param {Document} doc standardized clone
|
|
11
|
+
* @param {string} description meta description
|
|
12
|
+
* @returns {{ html: string, recovered: boolean }}
|
|
13
|
+
*/
|
|
14
|
+
export function recoverLeadParagraph(html: string, doc: Document, description: string): {
|
|
15
|
+
html: string;
|
|
16
|
+
recovered: boolean;
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* The heading for the capture: the page's own H1 when the document title or
|
|
20
|
+
* og:title contains it (and it is not just the site name); otherwise og:title;
|
|
21
|
+
* otherwise the title without a trailing site-name segment.
|
|
22
|
+
*
|
|
23
|
+
* @param {Document} doc standardized clone
|
|
24
|
+
* @param {{ documentTitle?: string, title?: string, siteName?: string, extractedTitle?: string }} [meta]
|
|
25
|
+
*/
|
|
26
|
+
export function chooseHeadline(doc: Document, meta?: {
|
|
27
|
+
documentTitle?: string;
|
|
28
|
+
title?: string;
|
|
29
|
+
siteName?: string;
|
|
30
|
+
extractedTitle?: string;
|
|
31
|
+
}): string;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/** @param {any} node */
|
|
2
|
+
export function isAdvertisementElement(node: any): boolean;
|
|
3
|
+
/**
|
|
4
|
+
* Creates an empty HTML document unconnected to the page: no scripts run and
|
|
5
|
+
* no custom elements upgrade in it.
|
|
6
|
+
* @param {Document} doc
|
|
7
|
+
*/
|
|
8
|
+
export function createInertDocument(doc: Document): Document;
|
|
9
|
+
/**
|
|
10
|
+
* Clones `root` (a Document or Element) into a new inert document.
|
|
11
|
+
* Open shadow roots replace their host's light children (native slots supply
|
|
12
|
+
* assigned children). Limits apply only to shadow content; a host over the
|
|
13
|
+
* limit is emptied and counted as an omission.
|
|
14
|
+
*
|
|
15
|
+
* @param {Document|Element} root
|
|
16
|
+
* @param {import("./types.js").SanitizeOptions} [options]
|
|
17
|
+
* @returns {{ document: Document, omissions: { shadowHosts: number } }}
|
|
18
|
+
*/
|
|
19
|
+
export function sanitizeToInertDocument(root: Document | Element, options?: import("./types.js").SanitizeOptions): {
|
|
20
|
+
document: Document;
|
|
21
|
+
omissions: {
|
|
22
|
+
shadowHosts: number;
|
|
23
|
+
};
|
|
24
|
+
};
|
|
25
|
+
/** @type {Readonly<Required<import("./types.js").SanitizeOptions>>} */
|
|
26
|
+
export const DEFAULT_SANITIZE_OPTIONS: Readonly<Required<import("./types.js").SanitizeOptions>>;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @param {Document} doc the inert clone (mutated in place)
|
|
3
|
+
* @param {{ url?: string }} [options]
|
|
4
|
+
*/
|
|
5
|
+
export function standardizeDocument(doc: Document, options?: {
|
|
6
|
+
url?: string;
|
|
7
|
+
}): {
|
|
8
|
+
lazyImagesResolved: number;
|
|
9
|
+
inlineImagesDropped: number;
|
|
10
|
+
scriptLinksUnwrapped: number;
|
|
11
|
+
emptyLinksRemoved: number;
|
|
12
|
+
preWrapped: number;
|
|
13
|
+
headingWidgetsRemoved: number;
|
|
14
|
+
headingsUnwrapped: number;
|
|
15
|
+
};
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/// <reference lib="dom" preserve="true" />
|
|
2
|
+
export type OutputMode = "full" | "no_links" | "text_only";
|
|
3
|
+
export type SanitizeOptions = {
|
|
4
|
+
stripScripts?: boolean | undefined;
|
|
5
|
+
stripStyles?: boolean | undefined;
|
|
6
|
+
stripNoscript?: boolean | undefined;
|
|
7
|
+
stripMeta?: boolean | undefined;
|
|
8
|
+
stripLinkTags?: boolean | undefined;
|
|
9
|
+
stripAds?: boolean | undefined;
|
|
10
|
+
/**
|
|
11
|
+
* app-specific overlays or widgets to drop, e.g. "[data-app-overlay]"
|
|
12
|
+
*/
|
|
13
|
+
extraStripSelectors?: readonly string[] | undefined;
|
|
14
|
+
shadowNodeLimit?: number | undefined;
|
|
15
|
+
shadowCharLimit?: number | undefined;
|
|
16
|
+
shadowDepthLimit?: number | undefined;
|
|
17
|
+
};
|
|
18
|
+
export type ReadabilityOptions = NonNullable<ConstructorParameters<typeof import("@mozilla/readability").Readability<string>>[1]>;
|
|
19
|
+
export type Extraction = import("./extract.js").Extraction;
|
|
20
|
+
export type Extractor = import("./extract.js").Extractor;
|
|
21
|
+
export type CollectedMetadata = ReturnType<typeof import("./metadata.js").collectMetadata>;
|
|
22
|
+
export type ExtractedMetadata = {
|
|
23
|
+
/**
|
|
24
|
+
* the heading chosen for the capture
|
|
25
|
+
*/
|
|
26
|
+
headline?: string | undefined;
|
|
27
|
+
/**
|
|
28
|
+
* the title the extractor reported
|
|
29
|
+
*/
|
|
30
|
+
extractedTitle?: string | undefined;
|
|
31
|
+
/**
|
|
32
|
+
* the byline the extractor reported
|
|
33
|
+
*/
|
|
34
|
+
byline?: string | undefined;
|
|
35
|
+
};
|
|
36
|
+
export type PageMarkdownMetadata = CollectedMetadata & ExtractedMetadata;
|
|
37
|
+
export type PageMarkdownExtraction = {
|
|
38
|
+
method: Extraction["method"];
|
|
39
|
+
/**
|
|
40
|
+
* why extraction fell back to the full page
|
|
41
|
+
*/
|
|
42
|
+
fallbackReason?: string | undefined;
|
|
43
|
+
/**
|
|
44
|
+
* a dropped lead paragraph was restored
|
|
45
|
+
*/
|
|
46
|
+
leadRecovered?: true | undefined;
|
|
47
|
+
/**
|
|
48
|
+
* how many facts `pageFacts` appended
|
|
49
|
+
*/
|
|
50
|
+
pageFacts?: number | undefined;
|
|
51
|
+
};
|
|
52
|
+
export type HiddenRunKind = "tag" | "variation_selector" | "invisible_operator" | "reserved_tag_plane";
|
|
53
|
+
/**
|
|
54
|
+
* A run as text-sanctifier's findHiddenRuns reports it.
|
|
55
|
+
*/
|
|
56
|
+
export type HiddenRun = {
|
|
57
|
+
kind: HiddenRunKind;
|
|
58
|
+
/**
|
|
59
|
+
* UTF-16 index of the first character
|
|
60
|
+
*/
|
|
61
|
+
offset: number;
|
|
62
|
+
/**
|
|
63
|
+
* UTF-16 index just past the last character
|
|
64
|
+
*/
|
|
65
|
+
end: number;
|
|
66
|
+
/**
|
|
67
|
+
* code points in the run
|
|
68
|
+
*/
|
|
69
|
+
count: number;
|
|
70
|
+
/**
|
|
71
|
+
* 1-based line of the first character
|
|
72
|
+
*/
|
|
73
|
+
line: number;
|
|
74
|
+
/**
|
|
75
|
+
* 1-based column of the first character, in UTF-16 units
|
|
76
|
+
*/
|
|
77
|
+
column: number;
|
|
78
|
+
/**
|
|
79
|
+
* tag runs only: the ASCII the tags spell
|
|
80
|
+
*/
|
|
81
|
+
decoded?: string | undefined;
|
|
82
|
+
/**
|
|
83
|
+
* tag runs only: true when `decoded` stops short of the run
|
|
84
|
+
*/
|
|
85
|
+
decodedTruncated?: boolean | undefined;
|
|
86
|
+
};
|
|
87
|
+
export type HiddenTextOmission = {
|
|
88
|
+
kind: `hidden_${HiddenRunKind}`;
|
|
89
|
+
/**
|
|
90
|
+
* runs removed from `markdown`
|
|
91
|
+
*/
|
|
92
|
+
count: number;
|
|
93
|
+
/**
|
|
94
|
+
* code points in those runs
|
|
95
|
+
*/
|
|
96
|
+
chars: number;
|
|
97
|
+
note: string;
|
|
98
|
+
};
|
|
99
|
+
/**
|
|
100
|
+
* Check `kind` before reading `chars`: only hidden-text omissions have it.
|
|
101
|
+
*/
|
|
102
|
+
export type PageMarkdownOmission = {
|
|
103
|
+
kind: "shadow_content";
|
|
104
|
+
count: number;
|
|
105
|
+
note: string;
|
|
106
|
+
} | {
|
|
107
|
+
kind: "json_line";
|
|
108
|
+
count: number;
|
|
109
|
+
note: string;
|
|
110
|
+
} | HiddenTextOmission;
|
|
111
|
+
export type StandardizationStats = ReturnType<typeof import("./standardize.js").standardizeDocument>;
|
|
112
|
+
export type PageMarkdownStats = {
|
|
113
|
+
mode: OutputMode;
|
|
114
|
+
extractedHtmlChars: number;
|
|
115
|
+
convertedChars: number;
|
|
116
|
+
markdownChars: number;
|
|
117
|
+
standardization: StandardizationStats;
|
|
118
|
+
};
|
|
119
|
+
export type PageMarkdownResult = {
|
|
120
|
+
/**
|
|
121
|
+
* the package version that made this result
|
|
122
|
+
*/
|
|
123
|
+
version: string;
|
|
124
|
+
markdown: string;
|
|
125
|
+
metadata: PageMarkdownMetadata;
|
|
126
|
+
extraction: PageMarkdownExtraction;
|
|
127
|
+
omissions: PageMarkdownOmission[];
|
|
128
|
+
warnings: string[];
|
|
129
|
+
stats: PageMarkdownStats;
|
|
130
|
+
/**
|
|
131
|
+
* present only with `includeFullPage`
|
|
132
|
+
*/
|
|
133
|
+
fullPageMarkdown?: string | undefined;
|
|
134
|
+
};
|
|
135
|
+
export type PageFact = {
|
|
136
|
+
label: string;
|
|
137
|
+
value: string;
|
|
138
|
+
};
|
|
139
|
+
export type TurndownFilter = string | string[] | ((node: HTMLElement, options: Record<string, unknown>) => boolean);
|
|
140
|
+
/**
|
|
141
|
+
* Turndown's public methods; Turndown ships no types of its own.
|
|
142
|
+
*/
|
|
143
|
+
export type TurndownInstance = {
|
|
144
|
+
turndown: (input: string | Node) => string;
|
|
145
|
+
use: (plugin: TurndownPlugin | TurndownPlugin[]) => TurndownInstance;
|
|
146
|
+
addRule: (key: string, rule: {
|
|
147
|
+
filter: TurndownFilter;
|
|
148
|
+
replacement: (content: string, node: Node, options: Record<string, unknown>) => string;
|
|
149
|
+
}) => TurndownInstance;
|
|
150
|
+
keep: (filter: TurndownFilter) => TurndownInstance;
|
|
151
|
+
remove: (filter: TurndownFilter) => TurndownInstance;
|
|
152
|
+
escape: (text: string) => string;
|
|
153
|
+
};
|
|
154
|
+
export type TurndownPlugin = (service: TurndownInstance) => void;
|
package/package.json
CHANGED
|
@@ -1,13 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "page-markdown",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.5",
|
|
4
4
|
"description": "Faithful page-to-Markdown for humans and agents: sanitize, extract the readable article, convert with GFM, and clean without losing content.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
7
|
-
".":
|
|
7
|
+
".": {
|
|
8
|
+
"types": "./dist/types/index.d.ts",
|
|
9
|
+
"default": "./src/index.js"
|
|
10
|
+
}
|
|
8
11
|
},
|
|
12
|
+
"types": "./dist/types/index.d.ts",
|
|
9
13
|
"files": [
|
|
10
14
|
"src",
|
|
15
|
+
"dist/types",
|
|
11
16
|
"LICENSE",
|
|
12
17
|
"ADDITIONAL_TERMS.md",
|
|
13
18
|
"THIRD_PARTY_NOTICES.md",
|
|
@@ -22,9 +27,15 @@
|
|
|
22
27
|
},
|
|
23
28
|
"scripts": {
|
|
24
29
|
"test": "node --test \"test/**/*.test.mjs\"",
|
|
30
|
+
"build:types": "tsc -p tsconfig.json",
|
|
31
|
+
"typecheck": "npm run build:types && node test/types/real-output.mjs && tsc -p test/types && tsc -p test/types/linkedom",
|
|
25
32
|
"size": "node bench/size.mjs",
|
|
26
|
-
"
|
|
27
|
-
"
|
|
33
|
+
"test:browser": "node test/browser/run.mjs",
|
|
34
|
+
"ci": "npm test && npm run typecheck && npm run test:browser && npm run size",
|
|
35
|
+
"prepack": "npm run build:types",
|
|
36
|
+
"prepublishOnly": "npm run ci",
|
|
37
|
+
"bench": "node bench/compare.mjs",
|
|
38
|
+
"bench:write": "node bench/compare.mjs --write"
|
|
28
39
|
},
|
|
29
40
|
"keywords": [
|
|
30
41
|
"markdown",
|
|
@@ -46,11 +57,12 @@
|
|
|
46
57
|
},
|
|
47
58
|
"dependencies": {
|
|
48
59
|
"@mozilla/readability": "0.6.0",
|
|
49
|
-
"text-sanctifier": "1.3.
|
|
60
|
+
"text-sanctifier": "1.3.8",
|
|
50
61
|
"turndown": "7.2.4"
|
|
51
62
|
},
|
|
52
63
|
"devDependencies": {
|
|
53
64
|
"esbuild": "0.27.0",
|
|
54
|
-
"linkedom": "0.18.12"
|
|
65
|
+
"linkedom": "0.18.12",
|
|
66
|
+
"typescript": "5.9.3"
|
|
55
67
|
}
|
|
56
68
|
}
|
package/src/clean.js
CHANGED
|
@@ -13,6 +13,7 @@ export function mapOutsideFences(markdown, transform) {
|
|
|
13
13
|
const blocks = [];
|
|
14
14
|
let lines = [];
|
|
15
15
|
let fence = null;
|
|
16
|
+
let indent = 0;
|
|
16
17
|
const flush = (isCode) => {
|
|
17
18
|
if (!lines.length) return;
|
|
18
19
|
const text = lines.join("\n");
|
|
@@ -20,12 +21,14 @@ export function mapOutsideFences(markdown, transform) {
|
|
|
20
21
|
lines = [];
|
|
21
22
|
};
|
|
22
23
|
for (const line of String(markdown).split("\n")) {
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
// Any indent or quote prefix: Turndown nests code in lists and blockquotes.
|
|
25
|
+
const match = line.match(/^([ >]*)(`{3,}|~{3,})(.*)$/);
|
|
26
|
+
if (!fence && match && !(match[2][0] === "`" && match[3].includes("`"))) {
|
|
25
27
|
flush(false);
|
|
26
|
-
fence = match[
|
|
28
|
+
fence = match[2];
|
|
29
|
+
indent = match[1].length;
|
|
27
30
|
lines.push(line);
|
|
28
|
-
} else if (fence && match && match[1][0] === fence[0] && match[
|
|
31
|
+
} else if (fence && match && match[1].length <= indent + 3 && match[2][0] === fence[0] && match[2].length >= fence.length && !match[3].trim()) {
|
|
29
32
|
lines.push(line);
|
|
30
33
|
flush(true);
|
|
31
34
|
fence = null;
|
|
@@ -81,8 +84,13 @@ export function cleanMarkdown(markdown) {
|
|
|
81
84
|
// Link text may hold escaped brackets and whole images: [](href)
|
|
82
85
|
// keeps the image, not "![alt" pointing at href.
|
|
83
86
|
const LINK_TARGET = String.raw`\((?:[^()\s]|\([^()]*\))+(?:\s+"[^"]*")?\)`;
|
|
84
|
-
const INLINE_LINK = new RegExp(String.raw`(!?)\[((?:\\.|!\[[^\]]*\]${LINK_TARGET}|[^\[\]\\])*)\]${LINK_TARGET}
|
|
85
|
-
|
|
87
|
+
const INLINE_LINK = new RegExp(String.raw`(!?)\[((?:\\.|!\[[^\]]*\]${LINK_TARGET}|[^\[\]\\])*)\]${LINK_TARGET}`);
|
|
88
|
+
// Replaces matches of `pattern`, leaving inline code spans as they are.
|
|
89
|
+
const replaceOutsideCode = (md, pattern, replace) => md.replace(
|
|
90
|
+
new RegExp(`\`+[^\`\\n]*\`+|${pattern.source}`, "g"),
|
|
91
|
+
(all, ...groups) => (all[0] === "`" ? all : replace(all, ...groups)),
|
|
92
|
+
);
|
|
93
|
+
const stripInlineLinks = (md) => replaceOutsideCode(md, INLINE_LINK, (all, bang, text) => (bang ? all : text));
|
|
86
94
|
const dropReferenceDefs = (md) => md.replace(/^\s{0,3}\[[^\]]+]:\s+\S+.*$/gm, "");
|
|
87
95
|
const stripStructure = (md) => md
|
|
88
96
|
.replace(/^\s{0,3}#{1,6}\s+/gm, "")
|
|
@@ -90,14 +98,18 @@ const stripStructure = (md) => md
|
|
|
90
98
|
.replace(/^\s*\d+\.\s+/gm, "")
|
|
91
99
|
.replace(/^\s{0,3}>\s?/gm, "")
|
|
92
100
|
.replace(/^\s*([-_*])\s*(?:\1\s*){2,}$/gm, "");
|
|
93
|
-
const stripEmphasis = (md) =>
|
|
94
|
-
|
|
101
|
+
const stripEmphasis = (md) => replaceOutsideCode(
|
|
102
|
+
replaceOutsideCode(md, /(\*\*|__)(.+?)\1/, (all, mark, text) => text),
|
|
103
|
+
/(?<![\w*])([*_])(?!\s)(.+?)(?<!\s)\1(?![\w*])/,
|
|
104
|
+
(all, mark, text) => text,
|
|
105
|
+
);
|
|
106
|
+
const stripImages = (md) => replaceOutsideCode(md, /!\[([^\]]*)\]\([^)]*\)/, (all, alt) => alt);
|
|
95
107
|
|
|
96
108
|
/**
|
|
97
109
|
* Output modes. "no_links" keeps link text. "text_only" also drops headings,
|
|
98
110
|
* list markers, emphasis and images. Both leave fenced code verbatim.
|
|
99
111
|
* @param {string} markdown
|
|
100
|
-
* @param {"
|
|
112
|
+
* @param {import("./types.js").OutputMode} [mode]
|
|
101
113
|
*/
|
|
102
114
|
export function applyMode(markdown, mode = "full") {
|
|
103
115
|
if (mode === "no_links") return mapOutsideFences(markdown, stripInlineLinks);
|
package/src/convert.js
CHANGED
|
@@ -65,25 +65,74 @@ function spanOf(cell) {
|
|
|
65
65
|
return Number.isFinite(span) && span > 1 ? Math.min(span, 50) : 1;
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
//
|
|
69
|
-
//
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
const
|
|
75
|
-
|
|
68
|
+
// Each cell's column, and the row-spanned cells that fill the slots before it
|
|
69
|
+
// (or at the row's end): null for the rest of a cell's column span. A span
|
|
70
|
+
// stops at its row group.
|
|
71
|
+
function tableGrid(rows) {
|
|
72
|
+
const columns = new Map();
|
|
73
|
+
const before = new Map();
|
|
74
|
+
const after = new Map();
|
|
75
|
+
let carry = [];
|
|
76
|
+
let group = null;
|
|
77
|
+
for (const row of rows) {
|
|
78
|
+
if (row.parentNode !== group) carry = [];
|
|
79
|
+
group = row.parentNode;
|
|
80
|
+
let column = 0;
|
|
81
|
+
let slots = [];
|
|
82
|
+
const take = (gaps) => {
|
|
83
|
+
while (carry[column]?.left > 0 || (gaps && column < carry.length)) {
|
|
84
|
+
const entry = carry[column];
|
|
85
|
+
slots.push(entry?.left > 0 ? entry.cell : null);
|
|
86
|
+
if (entry?.left > 0) entry.left -= 1;
|
|
87
|
+
column += 1;
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
for (const cell of rowCells(row)) {
|
|
91
|
+
slots = [];
|
|
92
|
+
take();
|
|
93
|
+
before.set(cell, slots);
|
|
94
|
+
columns.set(cell, column);
|
|
95
|
+
const left = Number.parseInt(cell.getAttribute?.("rowspan") || "1", 10) - 1;
|
|
96
|
+
const span = spanOf(cell);
|
|
97
|
+
for (let i = 0; i < span; i += 1) {
|
|
98
|
+
if (left > 0) carry[column + i] = { cell: i ? null : cell, left };
|
|
99
|
+
}
|
|
100
|
+
column += span;
|
|
101
|
+
}
|
|
102
|
+
slots = [];
|
|
103
|
+
take(true);
|
|
104
|
+
after.set(row, slots);
|
|
105
|
+
}
|
|
106
|
+
return { columns, before, after };
|
|
76
107
|
}
|
|
77
108
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
109
|
+
const tables = new WeakMap();
|
|
110
|
+
|
|
111
|
+
// Worked out once per table: cell and row rules run for every cell.
|
|
112
|
+
function tableInfo(table) {
|
|
113
|
+
let info = tables.get(table);
|
|
114
|
+
if (info) return info;
|
|
115
|
+
const rows = tableRows(table);
|
|
116
|
+
// Layout tables (nesting, a single cell, a single column) read better as
|
|
117
|
+
// blocks than as a grid.
|
|
118
|
+
const layout = Boolean(table.querySelector?.("table")) || !rows.length
|
|
119
|
+
|| Math.max(...rows.map((row) => rowCells(row).reduce((sum, cell) => sum + spanOf(cell), 0))) <= 1;
|
|
120
|
+
// A GFM cell is one line, so a data table holding code blocks is rendered as
|
|
121
|
+
// blocks too, each cell under its column's header: the code stays fenced.
|
|
122
|
+
const block = layout || Boolean(table.querySelector?.("pre"));
|
|
123
|
+
info = { layout, block, header: columnHeaders(rows[0]), text: new Map(), ...tableGrid(rows) };
|
|
124
|
+
tables.set(table, info);
|
|
125
|
+
return info;
|
|
82
126
|
}
|
|
83
127
|
|
|
84
128
|
function isBlockCell(node) {
|
|
85
129
|
const table = closest(node, "TABLE");
|
|
86
|
-
return !table ||
|
|
130
|
+
return !table || tableInfo(table).block;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// A row-spanned cell repeats in each row, so a row reads whole by itself.
|
|
134
|
+
function slotsMarkdown(slots, info) {
|
|
135
|
+
return slots.map((cell) => ` ${info.text.get(cell) || ""} |`).join("");
|
|
87
136
|
}
|
|
88
137
|
|
|
89
138
|
function labelText(cell) {
|
|
@@ -92,8 +141,7 @@ function labelText(cell) {
|
|
|
92
141
|
|
|
93
142
|
// A first row of two or more <th>: column headers. (A single spanning <th>
|
|
94
143
|
// is a title, not a header.)
|
|
95
|
-
function columnHeaders(
|
|
96
|
-
const first = tableRows(table)[0];
|
|
144
|
+
function columnHeaders(first) {
|
|
97
145
|
const cells = first ? rowCells(first) : [];
|
|
98
146
|
return cells.length > 1 && cells.every((cell) => cell.nodeName === "TH") ? first : null;
|
|
99
147
|
}
|
|
@@ -108,16 +156,13 @@ function rowHeader(row) {
|
|
|
108
156
|
// label; "" for layout tables and unlabelled cells.
|
|
109
157
|
function cellLabel(cell) {
|
|
110
158
|
const table = closest(cell, "TABLE");
|
|
111
|
-
|
|
159
|
+
const info = table && tableInfo(table);
|
|
160
|
+
if (!info || info.layout) return "";
|
|
112
161
|
const row = cell.parentNode;
|
|
113
|
-
const header =
|
|
162
|
+
const { header } = info;
|
|
114
163
|
if (header) {
|
|
115
164
|
if (row === header) return null;
|
|
116
|
-
|
|
117
|
-
for (const other of rowCells(row)) {
|
|
118
|
-
if (other === cell) break;
|
|
119
|
-
column += spanOf(other);
|
|
120
|
-
}
|
|
165
|
+
const column = info.columns.get(cell);
|
|
121
166
|
let start = 0;
|
|
122
167
|
for (const th of rowCells(header)) {
|
|
123
168
|
if (column < start + spanOf(th)) return labelText(th);
|
|
@@ -152,15 +197,19 @@ function tableRules(turndown) {
|
|
|
152
197
|
if (label === null) return "";
|
|
153
198
|
return label ? `\n\n**${createTurndown().escape(label)}**\n\n${content}\n\n` : `\n\n${content}\n\n`;
|
|
154
199
|
}
|
|
200
|
+
const info = tableInfo(closest(node, "TABLE"));
|
|
201
|
+
const text = cellText(content);
|
|
202
|
+
info.text.set(node, text);
|
|
155
203
|
const pad = " | ".repeat(spanOf(node) - 1);
|
|
156
|
-
return
|
|
204
|
+
return `${slotsMarkdown(info.before.get(node), info)} ${text}${pad} |`;
|
|
157
205
|
},
|
|
158
206
|
});
|
|
159
207
|
turndown.addRule("pmTableRow", {
|
|
160
208
|
filter: "tr",
|
|
161
209
|
replacement(content, node) {
|
|
162
210
|
if (isBlockCell(node)) return `\n${content}\n`;
|
|
163
|
-
|
|
211
|
+
const info = tableInfo(closest(node, "TABLE"));
|
|
212
|
+
return `\n|${content}${slotsMarkdown(info.after.get(node), info)}`;
|
|
164
213
|
},
|
|
165
214
|
});
|
|
166
215
|
turndown.addRule("pmTableSection", {
|
|
@@ -172,7 +221,7 @@ function tableRules(turndown) {
|
|
|
172
221
|
turndown.addRule("pmTable", {
|
|
173
222
|
filter: "table",
|
|
174
223
|
replacement(content, node) {
|
|
175
|
-
if (
|
|
224
|
+
if (tableInfo(node).block) return `\n\n${content}\n\n`;
|
|
176
225
|
const lines = content.split("\n").filter((line) => line.startsWith("|"));
|
|
177
226
|
if (!lines.length) return "\n\n";
|
|
178
227
|
const width = Math.max(...lines.map((line) => line.split(/(?<!\\)\|/).length - 2));
|
|
@@ -197,7 +246,10 @@ function tableRules(turndown) {
|
|
|
197
246
|
|
|
198
247
|
let shared = null;
|
|
199
248
|
|
|
200
|
-
/**
|
|
249
|
+
/**
|
|
250
|
+
* A configured Turndown instance (cached).
|
|
251
|
+
* @returns {import("./types.js").TurndownInstance}
|
|
252
|
+
*/
|
|
201
253
|
export function createTurndown() {
|
|
202
254
|
if (shared) return shared;
|
|
203
255
|
const turndown = new TurndownService({ ...TURNDOWN_OPTIONS });
|