@gmickel/gno 1.46.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +46 -0
- package/assets/skill/SKILL.md +7 -6
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +1 -0
- package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/browser-extension/dist/preview.html +1 -1
- package/browser-extension/dist/service-worker.js +32 -33
- package/bunfig.toml +2 -0
- package/package.json +40 -26
- package/spec/cli.md +21 -4
- package/spec/db/schema.sql +146 -1
- package/spec/mcp.md +26 -0
- package/src/app/context-runtime-types.ts +3 -0
- package/src/app/context-runtime.ts +2 -0
- package/src/cli/commands/ask.ts +6 -1
- package/src/cli/commands/daemon.ts +21 -8
- package/src/cli/commands/embed.ts +77 -41
- package/src/cli/detach.ts +3 -2
- package/src/config/types.ts +3 -3
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
- package/src/converters/adapters/officeparser/adapter.ts +1 -2
- package/src/converters/versions.ts +6 -8
- package/src/core/context-evidence.ts +8 -4
- package/src/core/job-manager.ts +95 -13
- package/src/core/network-boundary-inventory.ts +10 -0
- package/src/core/shutdown-budget.ts +45 -0
- package/src/embed/backlog.ts +107 -4
- package/src/embed/batch.ts +42 -2
- package/src/embed/fingerprint.ts +16 -0
- package/src/embed/retry.ts +113 -5
- package/src/embed/variant-backlog.ts +105 -0
- package/src/embed/variant-plan.ts +62 -0
- package/src/embed/variant-retry.ts +113 -0
- package/src/ingestion/graph-reconciliation.ts +327 -0
- package/src/ingestion/sync.ts +9 -272
- package/src/llm/http-inference.ts +6 -0
- package/src/llm/httpEmbedding.ts +37 -6
- package/src/llm/httpGeneration.ts +18 -3
- package/src/llm/httpRerank.ts +23 -5
- package/src/llm/inference-cancellation.ts +168 -0
- package/src/llm/inference-scope.ts +202 -0
- package/src/llm/lazy-ports.ts +115 -0
- package/src/llm/native-worker/client.ts +541 -0
- package/src/llm/native-worker/dispatcher.ts +228 -0
- package/src/llm/native-worker/embedding-identity.ts +33 -0
- package/src/llm/native-worker/entry.ts +173 -0
- package/src/llm/native-worker/errors.ts +32 -0
- package/src/llm/native-worker/evaluation.ts +16 -0
- package/src/llm/native-worker/owned-exit.ts +108 -0
- package/src/llm/native-worker/owner.ts +141 -0
- package/src/llm/native-worker/ports.ts +317 -0
- package/src/llm/native-worker/protocol.ts +442 -0
- package/src/llm/native-worker/runtime-config.ts +92 -0
- package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
- package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
- package/src/llm/nodeLlamaCpp/generation.ts +34 -5
- package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
- package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
- package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
- package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
- package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
- package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
- package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
- package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
- package/src/llm/types.ts +35 -5
- package/src/mcp/context.ts +27 -0
- package/src/mcp/http-transport.ts +12 -10
- package/src/mcp/server.ts +3 -0
- package/src/mcp/tool-profile.ts +30 -8
- package/src/mcp/tools/context.ts +8 -11
- package/src/mcp/tools/embed.ts +1 -1
- package/src/mcp/tools/index-cmd.ts +1 -1
- package/src/mcp/tools/index.ts +10 -8
- package/src/mcp/tools/query.ts +14 -30
- package/src/mcp/tools/vsearch.ts +1 -1
- package/src/pipeline/answer.ts +23 -3
- package/src/pipeline/claim-verifier.ts +6 -0
- package/src/pipeline/expansion.ts +43 -40
- package/src/pipeline/explain.ts +6 -2
- package/src/pipeline/filters.ts +63 -0
- package/src/pipeline/fusion.ts +29 -9
- package/src/pipeline/graph-retrieval.ts +29 -9
- package/src/pipeline/hybrid.ts +198 -55
- package/src/pipeline/hydration.ts +161 -0
- package/src/pipeline/owner-fusion.ts +87 -0
- package/src/pipeline/rerank.ts +35 -11
- package/src/pipeline/search.ts +13 -2
- package/src/pipeline/types.ts +5 -3
- package/src/pipeline/vsearch.ts +87 -7
- package/src/sdk/client.ts +47 -3
- package/src/sdk/embed.ts +63 -39
- package/src/serve/background-runtime.ts +1 -1
- package/src/serve/context.ts +41 -56
- package/src/serve/embed-scheduler.ts +58 -35
- package/src/serve/public/components/IndexingProgress.tsx +46 -60
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/shiki-language-ids.ts +14 -0
- package/src/serve/resident-admission.ts +36 -36
- package/src/serve/resident-background-work.ts +20 -2
- package/src/serve/resident-request.ts +11 -5
- package/src/serve/resident-runtime.ts +97 -61
- package/src/serve/resident-shutdown.ts +153 -0
- package/src/serve/routes/api.ts +3 -1
- package/src/serve/server.ts +47 -26
- package/src/store/migrations/028-vector-variants.ts +54 -0
- package/src/store/migrations/029-graph-reference-state.ts +77 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +251 -183
- package/src/store/sqlite/eligibility.ts +174 -0
- package/src/store/sqlite/graph-edge-application.ts +66 -0
- package/src/store/sqlite/graph-reference-state.ts +194 -0
- package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
- package/src/store/types.ts +80 -12
- package/src/store/vector/eligibility.ts +36 -0
- package/src/store/vector/freshness.ts +33 -6
- package/src/store/vector/lazy.ts +81 -0
- package/src/store/vector/sqlite-vec.ts +106 -54
- package/src/store/vector/stats.ts +14 -3
- package/src/store/vector/types.ts +35 -2
- package/src/store/vector/variant-search.ts +192 -0
- package/src/store/vector/variants.ts +451 -0
- package/vendor/converters/markitdown-ts/LICENSE +21 -0
- package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
- package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
- package/vendor/converters/markitdown-ts/package.json +77 -0
- package/vendor/converters/officeparser/LICENSE +21 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
- package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
- package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
- package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
- package/vendor/converters/officeparser/dist/cli.js +381 -0
- package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
- package/vendor/converters/officeparser/dist/defaults.js +218 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
- package/vendor/converters/officeparser/dist/index.d.ts +60 -0
- package/vendor/converters/officeparser/dist/index.js +72 -0
- package/vendor/converters/officeparser/dist/index.mjs +18 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
- package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
- package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
- package/vendor/converters/officeparser/dist/types.js +107 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
- package/vendor/converters/officeparser/package.json +147 -0
- package/vendor/converters/upstream-manifest.json +124 -0
- package/vendor/dependency-fixes/README.md +77 -0
- package/vendor/dependency-fixes/vendor-converters.py +83 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip.sha256 +0 -1
- package/browser-extension/dist/chunk-627emwpj.js +0 -75
- /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
|
@@ -0,0 +1,1287 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parseHtml = void 0;
|
|
4
|
+
const types_js_1 = require("../types.js");
|
|
5
|
+
const astUtils_js_1 = require("../utils/astUtils.js");
|
|
6
|
+
const errorUtils_js_1 = require("../utils/errorUtils.js");
|
|
7
|
+
const mathUtils_js_1 = require("../utils/mathUtils.js");
|
|
8
|
+
const sanitize_js_1 = require("../utils/sanitize.js");
|
|
9
|
+
/**
|
|
10
|
+
* Maximum element nesting depth accepted from an HTML/XHTML source before the parser gives up
|
|
11
|
+
* with a typed error rather than letting the recursion overflow the call stack. See the guard in
|
|
12
|
+
* `parseNode` for why this value and not a larger one.
|
|
13
|
+
*/
|
|
14
|
+
const MAX_HTML_NESTING_DEPTH = 256;
|
|
15
|
+
/**
|
|
16
|
+
* Decode the handful of HTML entities this parser leaves intact. Text nodes and attribute
|
|
17
|
+
* values are kept in their raw escaped form during parsing (see `parseAttributes`), so any
|
|
18
|
+
* branch that lifts text or an attribute into AST content has to decode first - `<` inside
|
|
19
|
+
* a code/math body is a less-than operator, not markup.
|
|
20
|
+
*/
|
|
21
|
+
const decodeEntities = (s) => s
|
|
22
|
+
.replace(/ /g, ' ')
|
|
23
|
+
.replace(/</g, '<')
|
|
24
|
+
.replace(/>/g, '>')
|
|
25
|
+
.replace(/&/g, '&')
|
|
26
|
+
.replace(/"/g, '"')
|
|
27
|
+
.replace(/'/g, "'");
|
|
28
|
+
/**
|
|
29
|
+
* Presents an `HtmlNode` as a `MathNode` for the shared MathML converter.
|
|
30
|
+
*
|
|
31
|
+
* The shapes already line up field for field; the one thing that must happen here is entity
|
|
32
|
+
* decoding, since this parser keeps text nodes in their raw escaped form and `<` inside an
|
|
33
|
+
* `<mo>` is a less-than operator, not markup.
|
|
34
|
+
*/
|
|
35
|
+
const toMathNode = (node) => ({
|
|
36
|
+
tagName: node.tagName,
|
|
37
|
+
attributes: node.attributes,
|
|
38
|
+
text: node.text === undefined ? undefined : decodeEntities(node.text),
|
|
39
|
+
children: (node.children || []).map(toMathNode),
|
|
40
|
+
});
|
|
41
|
+
const parseAttributes = (attrString) => {
|
|
42
|
+
const attrs = {};
|
|
43
|
+
// Attribute names follow the HTML5 rule - any character except whitespace and
|
|
44
|
+
// " ' > / = - rather than a hand-picked allowlist. The previous class
|
|
45
|
+
// ([a-zA-Z0-9\-:]) silently split a legal name on any character outside it, so
|
|
46
|
+
// `data_foo="x"` produced TWO attributes: `data` (empty) and an invented
|
|
47
|
+
// `foo="x"` that was never in the source. Harmless while nothing read unknown
|
|
48
|
+
// attributes; not harmless once they can be replayed into generated output.
|
|
49
|
+
const regex = /([^\s"'>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+)))?/g;
|
|
50
|
+
let match;
|
|
51
|
+
while ((match = regex.exec(attrString)) !== null) {
|
|
52
|
+
const name = match[1].toLowerCase();
|
|
53
|
+
const value = match[2] !== undefined ? match[2] : (match[3] !== undefined ? match[3] : (match[4] || ''));
|
|
54
|
+
attrs[name] = value;
|
|
55
|
+
}
|
|
56
|
+
return attrs;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* Splits an inline `style` attribute into a property -> value map.
|
|
60
|
+
*
|
|
61
|
+
* Replaces substring matching (`styleAttr.includes('font-weight: bold')`) and unanchored regexes
|
|
62
|
+
* (`/color:\s*([^;]+)/`), which were wrong in both directions:
|
|
63
|
+
* - false positives: `color:` matched inside `background-color:`, so
|
|
64
|
+
* `"background-color: red; color: blue"` yielded color=red - the *wrong* value, not merely a
|
|
65
|
+
* spurious one - and `width:` matched inside `max-width:`, so the ubiquitous responsive-image
|
|
66
|
+
* style `max-width: 100%` was read as an explicit width.
|
|
67
|
+
* - false negatives: `font-weight:bold` without a space, `font-weight: 700`, and `bolder` were
|
|
68
|
+
* all missed, as was `line-through` inside `text-decoration: underline line-through`.
|
|
69
|
+
*
|
|
70
|
+
* Splitting is quote- and paren-aware so a semicolon inside `url(data:image/png;base64,...)` or a
|
|
71
|
+
* quoted font stack doesn't shatter the declaration. `!important` is stripped from values, since
|
|
72
|
+
* substring matching used to tolerate it and exact comparison otherwise would not - dropping it
|
|
73
|
+
* would be a silent regression rather than the intended fix.
|
|
74
|
+
*/
|
|
75
|
+
const parseStyleDeclarations = (styleAttr) => {
|
|
76
|
+
const decls = new Map();
|
|
77
|
+
if (!styleAttr)
|
|
78
|
+
return decls;
|
|
79
|
+
let depth = 0;
|
|
80
|
+
let quote = null;
|
|
81
|
+
let current = '';
|
|
82
|
+
const chunks = [];
|
|
83
|
+
for (const ch of styleAttr) {
|
|
84
|
+
if (quote) {
|
|
85
|
+
if (ch === quote)
|
|
86
|
+
quote = null;
|
|
87
|
+
}
|
|
88
|
+
else if (ch === '"' || ch === '\'') {
|
|
89
|
+
quote = ch;
|
|
90
|
+
}
|
|
91
|
+
else if (ch === '(') {
|
|
92
|
+
depth++;
|
|
93
|
+
}
|
|
94
|
+
else if (ch === ')') {
|
|
95
|
+
if (depth > 0)
|
|
96
|
+
depth--;
|
|
97
|
+
}
|
|
98
|
+
else if (ch === ';' && depth === 0) {
|
|
99
|
+
chunks.push(current);
|
|
100
|
+
current = '';
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
current += ch;
|
|
104
|
+
}
|
|
105
|
+
chunks.push(current);
|
|
106
|
+
for (const chunk of chunks) {
|
|
107
|
+
const idx = chunk.indexOf(':');
|
|
108
|
+
if (idx === -1)
|
|
109
|
+
continue;
|
|
110
|
+
const prop = chunk.slice(0, idx).trim().toLowerCase();
|
|
111
|
+
if (!prop)
|
|
112
|
+
continue;
|
|
113
|
+
const value = chunk.slice(idx + 1).trim().replace(/\s*!\s*important\s*$/i, '').trim();
|
|
114
|
+
if (value)
|
|
115
|
+
decls.set(prop, value);
|
|
116
|
+
}
|
|
117
|
+
return decls;
|
|
118
|
+
};
|
|
119
|
+
/**
|
|
120
|
+
* Reads a declaration, also accepting the `-webkit-`/`-moz-`/`-ms-`/`-o-` prefixed spelling so a
|
|
121
|
+
* vendor-prefixed property keeps matching (substring matching used to catch those by accident).
|
|
122
|
+
*/
|
|
123
|
+
const getDeclaration = (decls, prop) => decls.get(prop)
|
|
124
|
+
?? decls.get(`-webkit-${prop}`)
|
|
125
|
+
?? decls.get(`-moz-${prop}`)
|
|
126
|
+
?? decls.get(`-ms-${prop}`)
|
|
127
|
+
?? decls.get(`-o-${prop}`);
|
|
128
|
+
/**
|
|
129
|
+
* Returns the first family from a `font-family` stack, respecting quotes so a quoted family name
|
|
130
|
+
* containing a comma (`'Fira, A', serif`) isn't split through the middle of its own name.
|
|
131
|
+
*/
|
|
132
|
+
const firstFontFamily = (fontFamily) => {
|
|
133
|
+
let quote = null;
|
|
134
|
+
let first = '';
|
|
135
|
+
for (const ch of fontFamily) {
|
|
136
|
+
if (quote) {
|
|
137
|
+
if (ch === quote) {
|
|
138
|
+
quote = null;
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
else if (ch === '"' || ch === '\'') {
|
|
143
|
+
quote = ch;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
else if (ch === ',') {
|
|
147
|
+
break;
|
|
148
|
+
}
|
|
149
|
+
first += ch;
|
|
150
|
+
}
|
|
151
|
+
return first.trim();
|
|
152
|
+
};
|
|
153
|
+
const parseHtmlTree = (html) => {
|
|
154
|
+
const root = { type: 'element', tagName: 'root', children: [], attributes: {} };
|
|
155
|
+
let current = root;
|
|
156
|
+
let cursor = 0;
|
|
157
|
+
while (cursor < html.length) {
|
|
158
|
+
const tagStart = html.indexOf('<', cursor);
|
|
159
|
+
if (tagStart === -1) {
|
|
160
|
+
const text = html.substring(cursor);
|
|
161
|
+
if (text)
|
|
162
|
+
current.children.push({ type: 'text', text, children: [], parent: current });
|
|
163
|
+
break;
|
|
164
|
+
}
|
|
165
|
+
if (tagStart > cursor) {
|
|
166
|
+
const text = html.substring(cursor, tagStart);
|
|
167
|
+
if (text)
|
|
168
|
+
current.children.push({ type: 'text', text, children: [], parent: current });
|
|
169
|
+
}
|
|
170
|
+
if (html.startsWith('<!--', tagStart)) {
|
|
171
|
+
const commentEnd = html.indexOf('-->', tagStart + 4);
|
|
172
|
+
cursor = commentEnd !== -1 ? commentEnd + 3 : html.length;
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
// Scan for the tag's closing '>', skipping any that appear inside a quoted attribute value.
|
|
176
|
+
// Browsers do NOT escape '>' inside attribute values on serialization, so a literal '>' there
|
|
177
|
+
// (e.g. a mermaid diagram's `-->` in data-mermaid) must not be read as the tag end. The scan
|
|
178
|
+
// is linear in the tag's length and the cursor never rewinds, so the parse stays O(n) overall
|
|
179
|
+
// (no substring().match allocation per '<').
|
|
180
|
+
let tagEndIdx = -1;
|
|
181
|
+
let attrQuote = '';
|
|
182
|
+
for (let i = tagStart + 1; i < html.length; i++) {
|
|
183
|
+
const ch = html[i];
|
|
184
|
+
if (attrQuote) {
|
|
185
|
+
if (ch === attrQuote)
|
|
186
|
+
attrQuote = '';
|
|
187
|
+
}
|
|
188
|
+
else if (ch === '"' || ch === '\'') {
|
|
189
|
+
attrQuote = ch;
|
|
190
|
+
}
|
|
191
|
+
else if (ch === '>') {
|
|
192
|
+
tagEndIdx = i;
|
|
193
|
+
break;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
if (tagEndIdx === -1) {
|
|
197
|
+
// The quote-aware scan ran to the end without closing the tag. That is almost always an
|
|
198
|
+
// unbalanced quote from a stray unescaped '<' in prose (e.g. "a < b's weight"), not a
|
|
199
|
+
// genuinely truncated tag. Retry naively for the next literal '>': the resulting
|
|
200
|
+
// pseudo-tag is then dropped, so a malformed run degrades exactly as it did before the
|
|
201
|
+
// quote-aware scan existed instead of swallowing the rest of the document into one text
|
|
202
|
+
// node. Well-formed input with balanced quotes never reaches here.
|
|
203
|
+
tagEndIdx = html.indexOf('>', tagStart);
|
|
204
|
+
}
|
|
205
|
+
if (tagEndIdx === -1) {
|
|
206
|
+
const text = html.substring(tagStart);
|
|
207
|
+
current.children.push({ type: 'text', text, children: [], parent: current });
|
|
208
|
+
break;
|
|
209
|
+
}
|
|
210
|
+
const tagContent = html.substring(tagStart + 1, tagEndIdx);
|
|
211
|
+
cursor = tagEndIdx + 1;
|
|
212
|
+
const isClosing = tagContent.startsWith('/');
|
|
213
|
+
const isSelfClosing = tagContent.endsWith('/');
|
|
214
|
+
const tagCore = tagContent.replace(/^\/|\/$/g, '').trim();
|
|
215
|
+
const firstSpace = tagCore.search(/\s/);
|
|
216
|
+
const tagName = (firstSpace === -1 ? tagCore : tagCore.substring(0, firstSpace)).toLowerCase();
|
|
217
|
+
const attrString = firstSpace === -1 ? '' : tagCore.substring(firstSpace);
|
|
218
|
+
if (!tagName || !tagName.match(/^[a-z0-9\-]+$/)) {
|
|
219
|
+
// Probably not a real tag, e.g., < 5
|
|
220
|
+
current.children.push({ type: 'text', text: `<${tagContent}>`, children: [], parent: current });
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
if (isClosing) {
|
|
224
|
+
let p = current;
|
|
225
|
+
while (p && p.tagName !== tagName) {
|
|
226
|
+
p = p.parent;
|
|
227
|
+
}
|
|
228
|
+
if (p && p.parent) {
|
|
229
|
+
current = p.parent;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
const node = {
|
|
234
|
+
type: 'element',
|
|
235
|
+
tagName,
|
|
236
|
+
attributes: parseAttributes(attrString),
|
|
237
|
+
children: [],
|
|
238
|
+
parent: current
|
|
239
|
+
};
|
|
240
|
+
current.children.push(node);
|
|
241
|
+
const voidElements = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr', '!doctype']);
|
|
242
|
+
if (!isSelfClosing && !voidElements.has(tagName)) {
|
|
243
|
+
current = node;
|
|
244
|
+
if (tagName === 'script' || tagName === 'style') {
|
|
245
|
+
// Case-insensitive search from `cursor` via a sticky-ish regex, instead of
|
|
246
|
+
// lower-casing the whole document on every <script>/<style> (was O(n^2)).
|
|
247
|
+
// tagName is validated to /^[a-z0-9-]+$/ above, so it's safe to interpolate.
|
|
248
|
+
const closeRe = new RegExp(`</${tagName}>`, 'gi');
|
|
249
|
+
closeRe.lastIndex = cursor;
|
|
250
|
+
const closeMatch = closeRe.exec(html);
|
|
251
|
+
if (closeMatch) {
|
|
252
|
+
node.children.push({
|
|
253
|
+
type: 'text',
|
|
254
|
+
text: html.substring(cursor, closeMatch.index),
|
|
255
|
+
children: [],
|
|
256
|
+
parent: node
|
|
257
|
+
});
|
|
258
|
+
cursor = closeMatch.index + closeMatch[0].length;
|
|
259
|
+
current = node.parent;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
return root;
|
|
266
|
+
};
|
|
267
|
+
const parseHtml = async (buffer, config) => {
|
|
268
|
+
// Honour cancellation requests before the HTML tree is built and traversed.
|
|
269
|
+
// The custom recursive HTML parser can be expensive for large documents;
|
|
270
|
+
// rejecting early here prevents both the parsing and the subsequent AST construction.
|
|
271
|
+
(0, errorUtils_js_1.checkAbortSignal)(config.abortSignal);
|
|
272
|
+
const textStr = buffer.toString('utf-8');
|
|
273
|
+
const root = parseHtmlTree(textStr);
|
|
274
|
+
// Find head and body
|
|
275
|
+
let head;
|
|
276
|
+
let body = root;
|
|
277
|
+
const findNode = (node, tag) => {
|
|
278
|
+
if (node.tagName === tag)
|
|
279
|
+
return node;
|
|
280
|
+
for (const child of node.children) {
|
|
281
|
+
const found = findNode(child, tag);
|
|
282
|
+
if (found)
|
|
283
|
+
return found;
|
|
284
|
+
}
|
|
285
|
+
return undefined;
|
|
286
|
+
};
|
|
287
|
+
const htmlNode = findNode(root, 'html');
|
|
288
|
+
if (htmlNode) {
|
|
289
|
+
head = findNode(htmlNode, 'head');
|
|
290
|
+
body = findNode(htmlNode, 'body') || htmlNode;
|
|
291
|
+
}
|
|
292
|
+
const metadata = {};
|
|
293
|
+
const attachments = [];
|
|
294
|
+
if (head) {
|
|
295
|
+
const titleNode = findNode(head, 'title');
|
|
296
|
+
if (titleNode && titleNode.children.length > 0 && titleNode.children[0].text) {
|
|
297
|
+
metadata.title = titleNode.children[0].text;
|
|
298
|
+
}
|
|
299
|
+
metadata.nativeProperties = {};
|
|
300
|
+
for (const child of head.children) {
|
|
301
|
+
if (child.tagName === 'meta') {
|
|
302
|
+
const name = child.attributes?.name || child.attributes?.property || child.attributes?.['http-equiv'];
|
|
303
|
+
if (name) {
|
|
304
|
+
metadata.nativeProperties[name] = child.attributes?.content || '';
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
const extractMeta = (name) => {
|
|
309
|
+
for (const child of head.children) {
|
|
310
|
+
if (child.tagName === 'meta' && (child.attributes?.name === name || child.attributes?.property === name)) {
|
|
311
|
+
return child.attributes?.content;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
return undefined;
|
|
315
|
+
};
|
|
316
|
+
const author = extractMeta('author');
|
|
317
|
+
if (author)
|
|
318
|
+
metadata.author = author;
|
|
319
|
+
const desc = extractMeta('description');
|
|
320
|
+
if (desc)
|
|
321
|
+
metadata.description = desc;
|
|
322
|
+
const created = extractMeta('dcterms.created');
|
|
323
|
+
if (created)
|
|
324
|
+
metadata.created = new Date(created);
|
|
325
|
+
const modified = extractMeta('dcterms.modified');
|
|
326
|
+
if (modified)
|
|
327
|
+
metadata.modified = new Date(modified);
|
|
328
|
+
const lastMod = extractMeta('lastModifiedBy');
|
|
329
|
+
if (lastMod)
|
|
330
|
+
metadata.lastModifiedBy = lastMod;
|
|
331
|
+
// Custom properties
|
|
332
|
+
const customProps = {};
|
|
333
|
+
for (const child of head.children) {
|
|
334
|
+
if (child.tagName === 'meta' && child.attributes?.name?.startsWith('custom:')) {
|
|
335
|
+
const key = child.attributes.name.substring(7);
|
|
336
|
+
const val = child.attributes.content || '';
|
|
337
|
+
// Try to infer type
|
|
338
|
+
if (val === 'true')
|
|
339
|
+
customProps[key] = true;
|
|
340
|
+
else if (val === 'false')
|
|
341
|
+
customProps[key] = false;
|
|
342
|
+
else if (!isNaN(Number(val)) && val.trim() !== '')
|
|
343
|
+
customProps[key] = Number(val);
|
|
344
|
+
else if (!isNaN(Date.parse(val)) && val.includes(':'))
|
|
345
|
+
customProps[key] = new Date(val);
|
|
346
|
+
else
|
|
347
|
+
customProps[key] = val;
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
if (Object.keys(customProps).length > 0)
|
|
351
|
+
metadata.customProperties = customProps;
|
|
352
|
+
}
|
|
353
|
+
const content = [];
|
|
354
|
+
let htmlListIdCounter = 1;
|
|
355
|
+
// Finds the checked state from a nested <input type="checkbox"> (GFM task-list items
|
|
356
|
+
// nest it inside a <label>, so it isn't a direct child of the <li>).
|
|
357
|
+
const findNestedCheckboxChecked = (n) => {
|
|
358
|
+
if (n.tagName === 'input' && (n.attributes?.type || '').toLowerCase() === 'checkbox') {
|
|
359
|
+
return 'checked' in (n.attributes || {});
|
|
360
|
+
}
|
|
361
|
+
for (const child of n.children) {
|
|
362
|
+
const found = findNestedCheckboxChecked(child);
|
|
363
|
+
if (found !== undefined)
|
|
364
|
+
return found;
|
|
365
|
+
}
|
|
366
|
+
return undefined;
|
|
367
|
+
};
|
|
368
|
+
// Populated from a <section data-footnotes> block (found and parsed before the main
|
|
369
|
+
// body loop, since references can appear anywhere earlier in the document) and
|
|
370
|
+
// consulted by parseChildren's <sup data-footnote-ref> handling below.
|
|
371
|
+
const footnoteDefinitions = new Map();
|
|
372
|
+
// Keys a `<sup data-footnote-ref>` actually consumed, so definitions in the section that no
|
|
373
|
+
// reference points at (orphans) can be recovered at the end instead of silently dropped.
|
|
374
|
+
const referencedFootnoteKeys = new Set();
|
|
375
|
+
// --- Generic attribute pass-through (htmlParserConfig.preserveAttributes) ---------------
|
|
376
|
+
// Captures attributes no typed metadata field consumed, so they can be replayed on
|
|
377
|
+
// generation. Everything here is a *defence-in-depth* filter: HtmlGenerator sanitizes again
|
|
378
|
+
// on the way out, because an AST can be built programmatically rather than parsed.
|
|
379
|
+
const preserveAttributes = config.htmlParserConfig?.preserveAttributes === true;
|
|
380
|
+
// `style` is already consumed wholesale into TextFormatting/metadata above, and `id` is
|
|
381
|
+
// consumed into anchorIds and re-emitted by the generator - carrying either would duplicate
|
|
382
|
+
// an attribute the generator composes itself. `class` is deliberately NOT excluded: the
|
|
383
|
+
// generator's class attribute is built purely from style-mapping and never from a parsed
|
|
384
|
+
// `class`, so without this a plain `<p class="lead">` loses "lead" entirely. The generator
|
|
385
|
+
// merges it into that attribute rather than emitting a second one.
|
|
386
|
+
const GENERATOR_OWNED_ATTRS = new Set(['id', 'style']);
|
|
387
|
+
/**
|
|
388
|
+
* Captures the attributes of `node` that `consumed` didn't claim.
|
|
389
|
+
* Returns undefined when nothing survives, so the field stays absent rather than `{}`.
|
|
390
|
+
*/
|
|
391
|
+
const collectHtmlAttributes = (node, consumed) => {
|
|
392
|
+
if (!preserveAttributes || !node.attributes)
|
|
393
|
+
return undefined;
|
|
394
|
+
const consumedSet = new Set(consumed.map(c => c.toLowerCase()));
|
|
395
|
+
const bag = {};
|
|
396
|
+
for (const [rawKey, value] of Object.entries(node.attributes)) {
|
|
397
|
+
const key = rawKey.toLowerCase();
|
|
398
|
+
if (consumedSet.has(key) || GENERATOR_OWNED_ATTRS.has(key))
|
|
399
|
+
continue;
|
|
400
|
+
// Event handlers are never carried, at any layer, with no opt-in.
|
|
401
|
+
if (/^on/i.test(key))
|
|
402
|
+
continue;
|
|
403
|
+
// srcdoc holds a whole HTML document; it cannot be safely escaped into an attribute.
|
|
404
|
+
if (key === 'srcdoc')
|
|
405
|
+
continue;
|
|
406
|
+
// Reject anything that isn't a plain attribute name outright - a key containing a
|
|
407
|
+
// quote or '=' is the shape an attribute-injection payload takes.
|
|
408
|
+
if (!(0, sanitize_js_1.isSafeHtmlAttributeName)(key))
|
|
409
|
+
continue;
|
|
410
|
+
bag[key] = value;
|
|
411
|
+
}
|
|
412
|
+
return Object.keys(bag).length > 0 ? bag : undefined;
|
|
413
|
+
};
|
|
414
|
+
const parseNode = (node, currentFormatting = {}, listContext, depth = 0) => {
|
|
415
|
+
// Guard against a maliciously deep element tree (e.g. tens of thousands of nested
|
|
416
|
+
// <div>) recursing until the call stack overflows.
|
|
417
|
+
//
|
|
418
|
+
// The previous limit of 1000 could never fire: measured overflow is around 800 and
|
|
419
|
+
// varies run to run (796/862/796 on three identical runs), so the RangeError always
|
|
420
|
+
// arrived first and the typed error this guard exists to produce never did. Failure was
|
|
421
|
+
// still graceful - it surfaces as a wrapped Error, not a crash - which is why this was a
|
|
422
|
+
// dead guard rather than a denial of service.
|
|
423
|
+
//
|
|
424
|
+
// 256 is chosen to hold across engines rather than tuned to one. It is far below the
|
|
425
|
+
// lowest overflow observed here and leaves room for a smaller frame budget on older V8
|
|
426
|
+
// (the supported floor is Node 18), while sitting orders of magnitude above real
|
|
427
|
+
// content: the bundled HTML and EPUB fixtures reach an AST depth of 8.
|
|
428
|
+
// Per node, alongside the depth guard: the two together are what make a hostile
|
|
429
|
+
// document both bounded and cancellable rather than only bounded.
|
|
430
|
+
(0, errorUtils_js_1.checkAbortSignal)(config.abortSignal);
|
|
431
|
+
if (depth > MAX_HTML_NESTING_DEPTH) {
|
|
432
|
+
throw (0, errorUtils_js_1.getOfficeError)(types_js_1.OfficeErrorType.MAX_NESTING_DEPTH_EXCEEDED);
|
|
433
|
+
}
|
|
434
|
+
if (node.type === 'text') {
|
|
435
|
+
let decodedText = decodeEntities(node.text || '');
|
|
436
|
+
if (!config.preserveXmlWhitespace) {
|
|
437
|
+
decodedText = decodedText.replace(/\s+/g, ' ');
|
|
438
|
+
}
|
|
439
|
+
if (!decodedText.trim() && !config.preserveXmlWhitespace)
|
|
440
|
+
return null;
|
|
441
|
+
const textNode = {
|
|
442
|
+
type: 'text',
|
|
443
|
+
text: decodedText,
|
|
444
|
+
formatting: Object.keys(currentFormatting).length > 0 ? { ...currentFormatting } : undefined
|
|
445
|
+
};
|
|
446
|
+
if (config.includeRawContent && node.text) {
|
|
447
|
+
// For text nodes in this manual parser, we just use the decoded text as raw content
|
|
448
|
+
// as we don't have accurate locators for the original source slice
|
|
449
|
+
textNode.rawContent = node.text;
|
|
450
|
+
}
|
|
451
|
+
return textNode;
|
|
452
|
+
}
|
|
453
|
+
if (node.type === 'element' && node.tagName) {
|
|
454
|
+
const tagName = node.tagName;
|
|
455
|
+
const newFormatting = { ...currentFormatting };
|
|
456
|
+
if (tagName === 'b' || tagName === 'strong')
|
|
457
|
+
newFormatting.bold = true;
|
|
458
|
+
if (tagName === 'i' || tagName === 'em')
|
|
459
|
+
newFormatting.italic = true;
|
|
460
|
+
if (tagName === 'u')
|
|
461
|
+
newFormatting.underline = true;
|
|
462
|
+
if (tagName === 'strike' || tagName === 's' || tagName === 'del')
|
|
463
|
+
newFormatting.strikethrough = true;
|
|
464
|
+
if (tagName === 'sub')
|
|
465
|
+
newFormatting.subscript = true;
|
|
466
|
+
if (tagName === 'sup')
|
|
467
|
+
newFormatting.superscript = true;
|
|
468
|
+
if (tagName === 'code')
|
|
469
|
+
newFormatting.font = 'monospace';
|
|
470
|
+
if (tagName === 'mark') {
|
|
471
|
+
// <mark> is a highlight. Use its data-color when present (an inline
|
|
472
|
+
// background-color style, read below, still wins); a bare <mark> falls back to the
|
|
473
|
+
// conventional yellow so it round-trips as a highlight rather than plain text.
|
|
474
|
+
newFormatting.backgroundColor = node.attributes?.['data-color'] || '#ffff00';
|
|
475
|
+
}
|
|
476
|
+
const styleAttr = node.attributes?.style || '';
|
|
477
|
+
const alignAttr = node.attributes?.align || '';
|
|
478
|
+
if (styleAttr || alignAttr) {
|
|
479
|
+
const decls = parseStyleDeclarations(styleAttr);
|
|
480
|
+
// `bold`, `bolder`, and any weight >= 600 are all bold; the old substring check
|
|
481
|
+
// only ever saw the literal "font-weight: bold".
|
|
482
|
+
const weight = getDeclaration(decls, 'font-weight');
|
|
483
|
+
if (weight) {
|
|
484
|
+
const numericWeight = parseInt(weight, 10);
|
|
485
|
+
if (weight === 'bold' || weight === 'bolder' || (!isNaN(numericWeight) && numericWeight >= 600)) {
|
|
486
|
+
newFormatting.bold = true;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
if (getDeclaration(decls, 'font-style') === 'italic')
|
|
490
|
+
newFormatting.italic = true;
|
|
491
|
+
// text-decoration is a shorthand that can carry several keywords at once, so
|
|
492
|
+
// "underline line-through" has to set both flags rather than only the first.
|
|
493
|
+
const decoration = getDeclaration(decls, 'text-decoration') ?? getDeclaration(decls, 'text-decoration-line');
|
|
494
|
+
if (decoration) {
|
|
495
|
+
const parts = decoration.split(/\s+/);
|
|
496
|
+
if (parts.includes('underline'))
|
|
497
|
+
newFormatting.underline = true;
|
|
498
|
+
if (parts.includes('line-through'))
|
|
499
|
+
newFormatting.strikethrough = true;
|
|
500
|
+
}
|
|
501
|
+
const color = decls.get('color');
|
|
502
|
+
if (color)
|
|
503
|
+
newFormatting.color = color;
|
|
504
|
+
const background = getDeclaration(decls, 'background-color');
|
|
505
|
+
if (background)
|
|
506
|
+
newFormatting.backgroundColor = background;
|
|
507
|
+
const size = getDeclaration(decls, 'font-size');
|
|
508
|
+
if (size)
|
|
509
|
+
newFormatting.size = size;
|
|
510
|
+
const fontFamily = getDeclaration(decls, 'font-family');
|
|
511
|
+
if (fontFamily)
|
|
512
|
+
newFormatting.font = firstFontFamily(fontFamily);
|
|
513
|
+
const textAlign = getDeclaration(decls, 'text-align')?.toLowerCase();
|
|
514
|
+
if (textAlign && ['left', 'center', 'right', 'justify'].includes(textAlign)) {
|
|
515
|
+
newFormatting.alignment = textAlign;
|
|
516
|
+
}
|
|
517
|
+
else if (alignAttr) {
|
|
518
|
+
const align = alignAttr.toLowerCase();
|
|
519
|
+
if (['left', 'center', 'right', 'justify'].includes(align)) {
|
|
520
|
+
newFormatting.alignment = align;
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
const anchorIds = node.attributes?.id ? [node.attributes.id] : [];
|
|
525
|
+
const parseChildren = (n, fmt, lCtx) => {
|
|
526
|
+
const kids = [];
|
|
527
|
+
for (const child of n.children) {
|
|
528
|
+
// Footnote/endnote reference: attach as .notes on the preceding node
|
|
529
|
+
// instead of inserting a visible node, matching WordParser's convention.
|
|
530
|
+
if (child.type === 'element' && child.tagName === 'sup' && child.attributes?.['data-footnote-ref'] !== undefined) {
|
|
531
|
+
const key = child.attributes['data-footnote-ref'];
|
|
532
|
+
referencedFootnoteKeys.add(key);
|
|
533
|
+
const definition = footnoteDefinitions.get(key);
|
|
534
|
+
const noteNode = {
|
|
535
|
+
type: 'note',
|
|
536
|
+
text: (definition || []).map(d => d.text || '').join(''),
|
|
537
|
+
children: definition || [],
|
|
538
|
+
metadata: { noteType: 'footnote', noteId: key }
|
|
539
|
+
};
|
|
540
|
+
if (kids.length > 0) {
|
|
541
|
+
const target = kids[kids.length - 1];
|
|
542
|
+
if (!target.notes)
|
|
543
|
+
target.notes = [];
|
|
544
|
+
target.notes.push(noteNode);
|
|
545
|
+
}
|
|
546
|
+
else {
|
|
547
|
+
kids.push({ type: 'text', text: '', notes: [noteNode] });
|
|
548
|
+
}
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
const parsed = parseNode(child, fmt, lCtx, depth + 1);
|
|
552
|
+
if (parsed) {
|
|
553
|
+
if (Array.isArray(parsed))
|
|
554
|
+
kids.push(...parsed);
|
|
555
|
+
else
|
|
556
|
+
kids.push(parsed);
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
return kids;
|
|
560
|
+
};
|
|
561
|
+
// Gated generic-iframe embed (HtmlGenerator's `gatedEmbeds` shape): an inert
|
|
562
|
+
// click-to-load placeholder that never auto-loads its src. Read it back to the same
|
|
563
|
+
// `embed` node unconditionally - capturing metadata is safe (the src is scheme-checked
|
|
564
|
+
// again on any re-emit); it is the trusted, already-gated counterpart to a raw <iframe>.
|
|
565
|
+
if (tagName === 'div' && node.attributes?.['data-embed-gated'] !== undefined) {
|
|
566
|
+
const gatedSrc = decodeEntities(node.attributes?.['data-embed-src'] || '');
|
|
567
|
+
if (!gatedSrc)
|
|
568
|
+
return null;
|
|
569
|
+
const gatedAlignAttr = node.attributes?.['data-embed-align'];
|
|
570
|
+
const gatedAlign = ['left', 'center', 'right'].includes(gatedAlignAttr) ? gatedAlignAttr : undefined;
|
|
571
|
+
const gatedNode = {
|
|
572
|
+
type: 'embed',
|
|
573
|
+
text: gatedSrc,
|
|
574
|
+
metadata: {
|
|
575
|
+
embedType: 'iframe',
|
|
576
|
+
url: gatedSrc,
|
|
577
|
+
width: node.attributes?.['data-embed-width'],
|
|
578
|
+
height: node.attributes?.['data-embed-height'],
|
|
579
|
+
align: gatedAlign,
|
|
580
|
+
label: node.attributes?.['data-embed-label']
|
|
581
|
+
}
|
|
582
|
+
};
|
|
583
|
+
if (config.includeRawContent)
|
|
584
|
+
gatedNode.rawContent = '<div data-embed-gated>...</div>';
|
|
585
|
+
return gatedNode;
|
|
586
|
+
}
|
|
587
|
+
// YouTube embeds: attribute-driven editors render
|
|
588
|
+
// <div data-youtube-video="ID" data-width="…" data-align="…">…<iframe…></div>.
|
|
589
|
+
// Recognise both the wrapper div and a bare iframe so externally-authored HTML
|
|
590
|
+
// (and a saved-then-reopened .md that fell back to raw HTML) both round-trip.
|
|
591
|
+
if (tagName === 'div' && node.attributes?.['data-youtube-video'] !== undefined) {
|
|
592
|
+
const videoId = node.attributes['data-youtube-video'] || '';
|
|
593
|
+
const width = node.attributes?.['data-width'];
|
|
594
|
+
const embedAlignAttr = node.attributes?.['data-align'];
|
|
595
|
+
const embedAlign = ['left', 'center', 'right'].includes(embedAlignAttr) ? embedAlignAttr : undefined;
|
|
596
|
+
const embedUrl = videoId ? `https://www.youtube.com/watch?v=${videoId}` : undefined;
|
|
597
|
+
const embedNode = {
|
|
598
|
+
type: 'embed',
|
|
599
|
+
// Childless nodes need .text so generic AST consumers (toText, chunking)
|
|
600
|
+
// don't silently drop them.
|
|
601
|
+
text: embedUrl,
|
|
602
|
+
metadata: {
|
|
603
|
+
embedType: 'youtube',
|
|
604
|
+
videoId,
|
|
605
|
+
url: embedUrl,
|
|
606
|
+
width,
|
|
607
|
+
align: embedAlign,
|
|
608
|
+
label: node.attributes?.['data-embed-label']
|
|
609
|
+
}
|
|
610
|
+
};
|
|
611
|
+
if (config.includeRawContent)
|
|
612
|
+
embedNode.rawContent = '<div data-youtube-video>...</div>';
|
|
613
|
+
return embedNode;
|
|
614
|
+
}
|
|
615
|
+
if (tagName === 'iframe') {
|
|
616
|
+
const src = node.attributes?.src || '';
|
|
617
|
+
const ytMatch = /youtube(?:-nocookie)?\.com/.test(src) ? src.match(/(?:embed\/|v=)([^&?/\s]+)/) : null;
|
|
618
|
+
if (ytMatch) {
|
|
619
|
+
const embedUrl = `https://www.youtube.com/watch?v=${ytMatch[1]}`;
|
|
620
|
+
const embedNode = {
|
|
621
|
+
type: 'embed',
|
|
622
|
+
text: embedUrl,
|
|
623
|
+
// Carry the iframe's own width/height so a YouTube iframe's dimensions are not
|
|
624
|
+
// dropped (metadata is unified across embedTypes; align has no source here).
|
|
625
|
+
metadata: {
|
|
626
|
+
embedType: 'youtube',
|
|
627
|
+
videoId: ytMatch[1],
|
|
628
|
+
url: embedUrl,
|
|
629
|
+
width: node.attributes?.width,
|
|
630
|
+
height: node.attributes?.height
|
|
631
|
+
}
|
|
632
|
+
};
|
|
633
|
+
if (config.includeRawContent)
|
|
634
|
+
embedNode.rawContent = '<iframe>...</iframe>';
|
|
635
|
+
return embedNode;
|
|
636
|
+
}
|
|
637
|
+
// Non-YouTube iframes are dropped by default (a deliberate security posture).
|
|
638
|
+
// preserveIframes opts back in, keeping the src as a generic 'iframe' embed; the
|
|
639
|
+
// src is scheme-checked again on generation, so this only widens what is retained.
|
|
640
|
+
// Decode the src (attribute values are stored entity-encoded) so it isn't
|
|
641
|
+
// double-escaped when the generator re-escapes it, which would corrupt query strings.
|
|
642
|
+
const decodedSrc = decodeEntities(src);
|
|
643
|
+
if ((0, sanitize_js_1.iframeAllowed)(decodedSrc, config.htmlParserConfig?.preserveIframes)) {
|
|
644
|
+
const iframeNode = {
|
|
645
|
+
type: 'embed',
|
|
646
|
+
text: decodedSrc,
|
|
647
|
+
metadata: {
|
|
648
|
+
embedType: 'iframe',
|
|
649
|
+
url: decodedSrc,
|
|
650
|
+
width: node.attributes?.width,
|
|
651
|
+
height: node.attributes?.height
|
|
652
|
+
}
|
|
653
|
+
};
|
|
654
|
+
if (config.includeRawContent)
|
|
655
|
+
iframeNode.rawContent = '<iframe>...</iframe>';
|
|
656
|
+
return iframeNode;
|
|
657
|
+
}
|
|
658
|
+
return null;
|
|
659
|
+
}
|
|
660
|
+
// Footnotes section: its definitions were already extracted up front (see
|
|
661
|
+
// footnoteDefinitions below), so skip it here wherever it appears in the tree -
|
|
662
|
+
// it isn't necessarily a direct child of <body> (e.g. it may be nested inside
|
|
663
|
+
// a non-standalone HtmlGenerator output's wrapping <div>).
|
|
664
|
+
if (tagName === 'section' && node.attributes?.['data-footnotes'] !== undefined) {
|
|
665
|
+
return null;
|
|
666
|
+
}
|
|
667
|
+
// Math. Two accepted shapes, disambiguated by the `data-math` value:
|
|
668
|
+
// 1. This library's own output - `data-math="inline|block"` names the mode, and the
|
|
669
|
+
// LaTeX is the visible ($-delimited, escaped) text content.
|
|
670
|
+
// 2. Attribute-driven producers that put the raw LaTeX in `data-math` and signal the
|
|
671
|
+
// mode through the class (`math-inline`/`math-block`) or the tag.
|
|
672
|
+
// Anything whose `data-math` is exactly `inline`/`block` takes path 1 unchanged; every
|
|
673
|
+
// other value is treated as LaTeX (path 2). LaTeX literally equal to `inline`/`block`
|
|
674
|
+
// is the only ambiguous input, and its text content wins there anyway.
|
|
675
|
+
if ((tagName === 'div' || tagName === 'span') && node.attributes?.['data-math'] !== undefined) {
|
|
676
|
+
const dataMath = node.attributes['data-math'];
|
|
677
|
+
const modeIsExplicit = dataMath === 'inline' || dataMath === 'block';
|
|
678
|
+
const classTokens = (node.attributes?.class || '').split(/\s+/);
|
|
679
|
+
const rawText = decodeEntities(node.children.map(c => c.text || '').join(''));
|
|
680
|
+
// Prefer the text content; fall back to the attribute value (path 2 producers may
|
|
681
|
+
// emit an empty body).
|
|
682
|
+
const source = rawText || (modeIsExplicit ? '' : decodeEntities(dataMath));
|
|
683
|
+
// Strip whichever `$`/`$$` delimiters are actually present, independent of the
|
|
684
|
+
// resolved mode - a `$`-delimited body inside a <div> must not keep its delimiters.
|
|
685
|
+
// The delimiter also disambiguates the mode when neither an explicit `data-math` nor
|
|
686
|
+
// a `math-inline`/`math-block` class settles it (so `<div data-math="x">$x$</div>`
|
|
687
|
+
// reads as inline, not block-via-tag).
|
|
688
|
+
let latex = source;
|
|
689
|
+
let delimiterMode;
|
|
690
|
+
if (source.length >= 4 && source.startsWith('$$') && source.endsWith('$$')) {
|
|
691
|
+
latex = source.slice(2, -2);
|
|
692
|
+
delimiterMode = 'block';
|
|
693
|
+
}
|
|
694
|
+
else if (source.length >= 2 && source.startsWith('$') && source.endsWith('$')) {
|
|
695
|
+
latex = source.slice(1, -1);
|
|
696
|
+
delimiterMode = 'inline';
|
|
697
|
+
}
|
|
698
|
+
const mathMode = modeIsExplicit
|
|
699
|
+
? dataMath
|
|
700
|
+
: classTokens.includes('math-block') ? 'block'
|
|
701
|
+
: classTokens.includes('math-inline') ? 'inline'
|
|
702
|
+
: delimiterMode ?? (tagName === 'div' ? 'block' : 'inline');
|
|
703
|
+
return {
|
|
704
|
+
type: 'code',
|
|
705
|
+
text: latex,
|
|
706
|
+
metadata: { math: mathMode }
|
|
707
|
+
};
|
|
708
|
+
}
|
|
709
|
+
// Native MathML. This is what a real-world page and every EPUB3 uses (EpubParser
|
|
710
|
+
// routes each spine item through here), as opposed to the `data-math` round-trip
|
|
711
|
+
// contract above, which only ever appears in this library's own HTML output. Without
|
|
712
|
+
// it, a `<math>` element fell through to the generic element handling below, which
|
|
713
|
+
// concatenates descendant text: `<mfrac><mn>1</mn><mn>2</mn></mfrac>` became "12".
|
|
714
|
+
if (tagName === 'math' || tagName.endsWith(':math')) {
|
|
715
|
+
// `display="block"` is MathML's own attribute for a display equation; the legacy
|
|
716
|
+
// `mode="display"` means the same thing and is still emitted by older producers.
|
|
717
|
+
const isBlock = node.attributes?.['display'] === 'block'
|
|
718
|
+
|| node.attributes?.['mode'] === 'display';
|
|
719
|
+
const latex = (0, mathUtils_js_1.mathmlTreeToLatex)(toMathNode(node));
|
|
720
|
+
if ((0, mathUtils_js_1.isEmptyMath)(latex))
|
|
721
|
+
return null;
|
|
722
|
+
return {
|
|
723
|
+
type: 'code',
|
|
724
|
+
text: latex,
|
|
725
|
+
metadata: { math: isBlock ? 'block' : 'inline' }
|
|
726
|
+
};
|
|
727
|
+
}
|
|
728
|
+
// Admonition: attribute-driven editors render
|
|
729
|
+
// <div class="admonition admonition-note" data-type="note">…children…</div>.
|
|
730
|
+
if (tagName === 'div' && (node.attributes?.class || '').split(/\s+/).includes('admonition')) {
|
|
731
|
+
const admonitionTypeAttr = node.attributes?.['data-type'];
|
|
732
|
+
const admonitionType = ['note', 'tip', 'important', 'warning', 'caution'].includes(admonitionTypeAttr)
|
|
733
|
+
? admonitionTypeAttr
|
|
734
|
+
: 'note';
|
|
735
|
+
const admonitionNode = {
|
|
736
|
+
type: 'admonition',
|
|
737
|
+
metadata: { admonitionType },
|
|
738
|
+
children: parseChildren(node, newFormatting, listContext)
|
|
739
|
+
};
|
|
740
|
+
if (config.includeRawContent)
|
|
741
|
+
admonitionNode.rawContent = '<div class="admonition">...</div>';
|
|
742
|
+
return admonitionNode;
|
|
743
|
+
}
|
|
744
|
+
// Blockquote. Previously dropped entirely (its children were lifted out unquoted), so a
|
|
745
|
+
// <blockquote> lost its `> ` on the Markdown hop. Mark each block child with the 'Quote'
|
|
746
|
+
// style the styleMapper maps back to a blockquote; loose inline content is wrapped in one
|
|
747
|
+
// Quote-styled paragraph so it isn't emitted as an ordinary line.
|
|
748
|
+
if (tagName === 'blockquote') {
|
|
749
|
+
const kids = parseChildren(node, newFormatting, listContext);
|
|
750
|
+
const isBlock = (t) => t === 'paragraph' || t === 'heading' || t === 'list';
|
|
751
|
+
if (!kids.some(k => isBlock(k.type))) {
|
|
752
|
+
return { type: 'paragraph', metadata: { style: 'Quote' }, children: kids };
|
|
753
|
+
}
|
|
754
|
+
kids.forEach(k => {
|
|
755
|
+
if (isBlock(k.type))
|
|
756
|
+
k.metadata = { ...k.metadata, style: 'Quote' };
|
|
757
|
+
});
|
|
758
|
+
return kids;
|
|
759
|
+
}
|
|
760
|
+
// Mermaid diagrams. Attribute-driven producers render a
|
|
761
|
+
// <div class="mermaid" data-mermaid="<code>"> with the code also as text content.
|
|
762
|
+
// Map either shape to a fenced code node with language `mermaid`, so it round-trips as
|
|
763
|
+
// a ```mermaid block. Previously this div fell through to generic handling and its
|
|
764
|
+
// code flattened to paragraph text.
|
|
765
|
+
if (tagName === 'div' && (node.attributes?.['data-mermaid'] !== undefined || (node.attributes?.class || '').split(/\s+/).includes('mermaid'))) {
|
|
766
|
+
const code = decodeEntities(node.children.map(c => c.text || '').join('')).trim()
|
|
767
|
+
|| decodeEntities(node.attributes?.['data-mermaid'] || '');
|
|
768
|
+
// Only claim this as a mermaid code node when there is actual diagram source.
|
|
769
|
+
// A bare `class="mermaid"` div with nested elements (a mermaid.js-rendered <svg>,
|
|
770
|
+
// or a div merely reusing the class for styling) has no direct text and no
|
|
771
|
+
// data-mermaid; fall through to generic handling so its content is not dropped.
|
|
772
|
+
if (code) {
|
|
773
|
+
const mermaidNode = {
|
|
774
|
+
type: 'code',
|
|
775
|
+
text: code,
|
|
776
|
+
metadata: { language: 'mermaid' }
|
|
777
|
+
};
|
|
778
|
+
if (config.includeRawContent)
|
|
779
|
+
mermaidNode.rawContent = '<div data-mermaid>...</div>';
|
|
780
|
+
return mermaidNode;
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
// Skip structural containers produced by HtmlGenerator to avoid deep AST nesting
|
|
784
|
+
if (tagName === 'div' && (node.attributes?.class === 'container' ||
|
|
785
|
+
node.attributes?.class === 'spreadsheet-container' ||
|
|
786
|
+
node.attributes?.class === 'presentation-container' ||
|
|
787
|
+
node.attributes?.class === 'pdf-container' ||
|
|
788
|
+
node.attributes?.class === 'metadata-summary' ||
|
|
789
|
+
node.attributes?.class === 'image-container' ||
|
|
790
|
+
node.attributes?.class === 'chart-container' ||
|
|
791
|
+
node.attributes?.class === 'table-container' ||
|
|
792
|
+
node.attributes?.class === 'caption' ||
|
|
793
|
+
node.attributes?.class === 'sheet' ||
|
|
794
|
+
node.attributes?.class === 'page' ||
|
|
795
|
+
node.attributes?.class === 'slide' ||
|
|
796
|
+
node.attributes?.class === 'note-content')) {
|
|
797
|
+
return parseChildren(node, newFormatting, listContext);
|
|
798
|
+
}
|
|
799
|
+
if (tagName === 'article') {
|
|
800
|
+
return parseChildren(node, newFormatting, listContext);
|
|
801
|
+
}
|
|
802
|
+
if (tagName === 'p' || tagName === 'div') {
|
|
803
|
+
const children = parseChildren(node, newFormatting, listContext);
|
|
804
|
+
// If it's a div and contains block elements, return children directly
|
|
805
|
+
const hasBlockElements = children.some(c => ['paragraph', 'table', 'heading', 'list', 'image', 'chart', 'code', 'embed', 'admonition', 'definitionList'].includes(c.type));
|
|
806
|
+
if (tagName === 'div' && hasBlockElements) {
|
|
807
|
+
return children;
|
|
808
|
+
}
|
|
809
|
+
// Flatten nested paragraphs to avoid deep AST nesting (e.g. from notes)
|
|
810
|
+
const flattenedChildren = [];
|
|
811
|
+
for (const child of children) {
|
|
812
|
+
if (child.type === 'paragraph' && child.children) {
|
|
813
|
+
flattenedChildren.push(...child.children);
|
|
814
|
+
}
|
|
815
|
+
else {
|
|
816
|
+
flattenedChildren.push(child);
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
const pNode = {
|
|
820
|
+
type: 'paragraph',
|
|
821
|
+
metadata: { alignment: newFormatting.alignment, anchorIds: anchorIds.length > 0 ? anchorIds : undefined },
|
|
822
|
+
children: flattenedChildren,
|
|
823
|
+
htmlAttributes: collectHtmlAttributes(node, ['align'])
|
|
824
|
+
};
|
|
825
|
+
if (config.includeRawContent) {
|
|
826
|
+
// Note: Since this is a manual parser without locators, we can't easily get the original source slice.
|
|
827
|
+
// We'll skip rawContent for structural nodes here unless we want to implement index tracking in parseHtmlTree.
|
|
828
|
+
}
|
|
829
|
+
return pNode;
|
|
830
|
+
}
|
|
831
|
+
if (tagName.match(/^h[1-6]$/)) {
|
|
832
|
+
const level = parseInt(tagName.substring(1));
|
|
833
|
+
const hNode = {
|
|
834
|
+
type: 'heading',
|
|
835
|
+
metadata: { level, alignment: newFormatting.alignment, anchorIds: anchorIds.length > 0 ? anchorIds : undefined },
|
|
836
|
+
children: parseChildren(node, newFormatting, listContext),
|
|
837
|
+
htmlAttributes: collectHtmlAttributes(node, ['align'])
|
|
838
|
+
};
|
|
839
|
+
return hNode;
|
|
840
|
+
}
|
|
841
|
+
if (tagName === 'dl') {
|
|
842
|
+
return {
|
|
843
|
+
type: 'definitionList',
|
|
844
|
+
children: parseChildren(node, newFormatting, listContext)
|
|
845
|
+
};
|
|
846
|
+
}
|
|
847
|
+
if (tagName === 'dt') {
|
|
848
|
+
return {
|
|
849
|
+
type: 'definitionTerm',
|
|
850
|
+
children: parseChildren(node, newFormatting, listContext)
|
|
851
|
+
};
|
|
852
|
+
}
|
|
853
|
+
if (tagName === 'dd') {
|
|
854
|
+
return {
|
|
855
|
+
type: 'definitionDescription',
|
|
856
|
+
children: parseChildren(node, newFormatting, listContext)
|
|
857
|
+
};
|
|
858
|
+
}
|
|
859
|
+
if (tagName === 'abbr') {
|
|
860
|
+
const title = node.attributes?.title;
|
|
861
|
+
const children = parseChildren(node, newFormatting, listContext);
|
|
862
|
+
if (title) {
|
|
863
|
+
children.forEach(c => {
|
|
864
|
+
if (c.type === 'text') {
|
|
865
|
+
c.metadata = { ...c.metadata, abbreviationTitle: title };
|
|
866
|
+
}
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
return children;
|
|
870
|
+
}
|
|
871
|
+
if (tagName === 'cite' && node.attributes?.['data-citation-key'] !== undefined) {
|
|
872
|
+
const citationKey = node.attributes['data-citation-key'];
|
|
873
|
+
return {
|
|
874
|
+
type: 'text',
|
|
875
|
+
text: citationKey,
|
|
876
|
+
formatting: Object.keys(newFormatting).length > 0 ? { ...newFormatting } : undefined,
|
|
877
|
+
metadata: { citationKey }
|
|
878
|
+
};
|
|
879
|
+
}
|
|
880
|
+
// Attribute-driven citation shape: a <span> carrying the `citation` class token (among
|
|
881
|
+
// any others) and a non-empty data-key. Produces the same bare-key text node as the
|
|
882
|
+
// <cite> form above. An empty/absent data-key falls through to generic span handling,
|
|
883
|
+
// so the span's visible text still survives.
|
|
884
|
+
if (tagName === 'span'
|
|
885
|
+
&& (node.attributes?.class || '').split(/\s+/).includes('citation')
|
|
886
|
+
&& node.attributes?.['data-key']) {
|
|
887
|
+
const citationKey = decodeEntities(node.attributes['data-key']);
|
|
888
|
+
return {
|
|
889
|
+
type: 'text',
|
|
890
|
+
text: citationKey,
|
|
891
|
+
formatting: Object.keys(newFormatting).length > 0 ? { ...newFormatting } : undefined,
|
|
892
|
+
metadata: { citationKey }
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
if (tagName === 'ul' || tagName === 'ol') {
|
|
896
|
+
const isNewTopLevel = !listContext;
|
|
897
|
+
const newListContext = {
|
|
898
|
+
listId: isNewTopLevel ? `html-list-${htmlListIdCounter++}` : listContext.listId,
|
|
899
|
+
type: tagName === 'ol' ? 'ordered' : 'unordered',
|
|
900
|
+
level: isNewTopLevel ? 0 : listContext.level + 1,
|
|
901
|
+
counters: isNewTopLevel ? {} : { ...listContext.counters }, // Clone to avoid side effects on parent levels
|
|
902
|
+
isTask: node.attributes?.['data-type'] === 'taskList'
|
|
903
|
+
};
|
|
904
|
+
// Initialize counter for this level
|
|
905
|
+
if (tagName === 'ol' && node.attributes?.start) {
|
|
906
|
+
const start = parseInt(node.attributes.start, 10);
|
|
907
|
+
newListContext.counters[newListContext.level] = isNaN(start) ? 0 : start - 1;
|
|
908
|
+
}
|
|
909
|
+
else {
|
|
910
|
+
newListContext.counters[newListContext.level] = 0;
|
|
911
|
+
}
|
|
912
|
+
return parseChildren(node, currentFormatting, newListContext);
|
|
913
|
+
}
|
|
914
|
+
if (tagName === 'li') {
|
|
915
|
+
if (listContext) {
|
|
916
|
+
if (node.attributes?.value) {
|
|
917
|
+
const val = parseInt(node.attributes.value, 10);
|
|
918
|
+
if (!isNaN(val))
|
|
919
|
+
listContext.counters[listContext.level] = val;
|
|
920
|
+
}
|
|
921
|
+
else {
|
|
922
|
+
listContext.counters[listContext.level]++;
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
const children = parseChildren(node, newFormatting, listContext);
|
|
926
|
+
const nestedLists = children.filter(c => c.type === 'list');
|
|
927
|
+
const selfChildren = children.filter(c => c.type !== 'list');
|
|
928
|
+
let isTask;
|
|
929
|
+
let checked;
|
|
930
|
+
if (listContext?.isTask) {
|
|
931
|
+
isTask = true;
|
|
932
|
+
const dataChecked = node.attributes?.['data-checked'];
|
|
933
|
+
checked = dataChecked !== undefined ? dataChecked === 'true' : (findNestedCheckboxChecked(node) ?? false);
|
|
934
|
+
}
|
|
935
|
+
const selfNode = {
|
|
936
|
+
type: 'list',
|
|
937
|
+
text: selfChildren.map(c => c.text || '').join(''),
|
|
938
|
+
metadata: {
|
|
939
|
+
listType: listContext?.type || 'unordered',
|
|
940
|
+
indentation: listContext?.level || 0,
|
|
941
|
+
alignment: newFormatting.alignment || 'left',
|
|
942
|
+
listId: listContext?.listId || 'html-list-none',
|
|
943
|
+
itemIndex: (listContext?.counters[listContext.level] ?? 1) - 1,
|
|
944
|
+
anchorIds: anchorIds.length > 0 ? anchorIds : undefined,
|
|
945
|
+
isTask,
|
|
946
|
+
checked
|
|
947
|
+
},
|
|
948
|
+
children: selfChildren
|
|
949
|
+
};
|
|
950
|
+
return [selfNode, ...nestedLists];
|
|
951
|
+
}
|
|
952
|
+
if (tagName === 'table') {
|
|
953
|
+
// Attribute-driven editors render data-align on the <table> itself.
|
|
954
|
+
const tableAlignAttr = node.attributes?.['data-align'];
|
|
955
|
+
const tableAlign = ['left', 'center', 'right'].includes(tableAlignAttr) ? tableAlignAttr : undefined;
|
|
956
|
+
const tableNode = {
|
|
957
|
+
type: 'table',
|
|
958
|
+
metadata: { anchorIds: anchorIds.length > 0 ? anchorIds : undefined, align: tableAlign },
|
|
959
|
+
children: parseChildren(node, newFormatting, listContext),
|
|
960
|
+
htmlAttributes: collectHtmlAttributes(node, ['data-align', 'align'])
|
|
961
|
+
};
|
|
962
|
+
if (config.includeRawContent) {
|
|
963
|
+
tableNode.rawContent = '<table>...</table>';
|
|
964
|
+
}
|
|
965
|
+
return tableNode;
|
|
966
|
+
}
|
|
967
|
+
if (tagName === 'tr') {
|
|
968
|
+
const rowNode = {
|
|
969
|
+
type: 'row',
|
|
970
|
+
children: parseChildren(node, newFormatting, listContext),
|
|
971
|
+
htmlAttributes: collectHtmlAttributes(node, [])
|
|
972
|
+
};
|
|
973
|
+
if (config.includeRawContent) {
|
|
974
|
+
rowNode.rawContent = '<tr>...</tr>';
|
|
975
|
+
}
|
|
976
|
+
return rowNode;
|
|
977
|
+
}
|
|
978
|
+
if (tagName === 'td' || tagName === 'th') {
|
|
979
|
+
// Merged cells: mirrors the colspan/rowspan reading already done in
|
|
980
|
+
// MarkdownParser's inline HTML-table handler.
|
|
981
|
+
const colSpanAttr = node.attributes?.colspan;
|
|
982
|
+
const rowSpanAttr = node.attributes?.rowspan;
|
|
983
|
+
const colSpan = colSpanAttr ? parseInt(colSpanAttr, 10) : undefined;
|
|
984
|
+
const rowSpan = rowSpanAttr ? parseInt(rowSpanAttr, 10) : undefined;
|
|
985
|
+
// Per-column GFM alignment: read the cell's own `text-align` (or a legacy `align=`
|
|
986
|
+
// attribute) into `CellMetadata.align`, so the `:---`/`:---:`/`---:` markers survive
|
|
987
|
+
// AST -> HTML -> AST. `justify` has no pipe-table marker, so it is not a cell align.
|
|
988
|
+
// The table-level `<table data-align>` form is read separately in the `table` branch.
|
|
989
|
+
const cellTextAlign = (getDeclaration(parseStyleDeclarations(node.attributes?.style || ''), 'text-align')
|
|
990
|
+
|| node.attributes?.align || '').toLowerCase();
|
|
991
|
+
const cellAlign = ['left', 'center', 'right'].includes(cellTextAlign)
|
|
992
|
+
? cellTextAlign
|
|
993
|
+
: undefined;
|
|
994
|
+
const cellNode = {
|
|
995
|
+
type: 'cell',
|
|
996
|
+
metadata: {
|
|
997
|
+
colSpan: colSpan && !isNaN(colSpan) ? colSpan : undefined,
|
|
998
|
+
rowSpan: rowSpan && !isNaN(rowSpan) ? rowSpan : undefined,
|
|
999
|
+
align: cellAlign
|
|
1000
|
+
},
|
|
1001
|
+
children: parseChildren(node, newFormatting, listContext),
|
|
1002
|
+
htmlAttributes: collectHtmlAttributes(node, ['colspan', 'rowspan', 'align'])
|
|
1003
|
+
};
|
|
1004
|
+
if (config.includeRawContent) {
|
|
1005
|
+
cellNode.rawContent = '<td>...</td>';
|
|
1006
|
+
}
|
|
1007
|
+
return cellNode;
|
|
1008
|
+
}
|
|
1009
|
+
if (tagName === 'img') {
|
|
1010
|
+
const src = node.attributes?.src;
|
|
1011
|
+
const alt = node.attributes?.alt;
|
|
1012
|
+
// Attribute-driven editors render data-width/data-align, falling back to
|
|
1013
|
+
// parsing the inline style for consumers that only emit the CSS.
|
|
1014
|
+
const imgDecls = parseStyleDeclarations(node.attributes?.style || '');
|
|
1015
|
+
// Exact lookup, so `max-width: 100%` - the standard responsive-image style, and by
|
|
1016
|
+
// far the most common inline style on an <img> - is no longer read as a declared
|
|
1017
|
+
// width. It constrains the rendered size; it is not an author-specified width.
|
|
1018
|
+
const width = node.attributes?.['data-width'] || getDeclaration(imgDecls, 'width');
|
|
1019
|
+
// Alignment is inferred from which auto margin is present. Comparing the parsed
|
|
1020
|
+
// value rather than substring-matching "margin-left: 0" stops `margin-left: 0.5rem`
|
|
1021
|
+
// from being read as left-aligned, and lets the `margin: 0 auto` centering
|
|
1022
|
+
// shorthand be recognised at all.
|
|
1023
|
+
const marginLeft = getDeclaration(imgDecls, 'margin-left');
|
|
1024
|
+
const marginRight = getDeclaration(imgDecls, 'margin-right');
|
|
1025
|
+
const marginShorthand = getDeclaration(imgDecls, 'margin');
|
|
1026
|
+
const isZero = (v) => v !== undefined && /^0(?:[a-z%]*)$/.test(v);
|
|
1027
|
+
const shorthandParts = marginShorthand ? marginShorthand.split(/\s+/) : [];
|
|
1028
|
+
const shorthandCentres = shorthandParts.length > 1
|
|
1029
|
+
&& shorthandParts[shorthandParts.length - 1] === 'auto'
|
|
1030
|
+
&& shorthandParts[1] === 'auto';
|
|
1031
|
+
const alignAttr = node.attributes?.['data-align']
|
|
1032
|
+
?? (shorthandCentres ? 'center'
|
|
1033
|
+
: (isZero(marginLeft) && !isZero(marginRight) ? 'left'
|
|
1034
|
+
: (isZero(marginRight) && !isZero(marginLeft) ? 'right' : undefined)));
|
|
1035
|
+
const align = ['left', 'center', 'right'].includes(alignAttr) ? alignAttr : undefined;
|
|
1036
|
+
let imageNode;
|
|
1037
|
+
if (src?.startsWith('data:')) {
|
|
1038
|
+
const match = src.match(/^data:([^;]+);base64,(.*)$/);
|
|
1039
|
+
if (match && config.extractAttachments) {
|
|
1040
|
+
const mimeType = match[1];
|
|
1041
|
+
const data = match[2];
|
|
1042
|
+
const name = `image_${attachments.length + 1}.${mimeType.split('/')[1]}`;
|
|
1043
|
+
attachments.push({
|
|
1044
|
+
type: 'image',
|
|
1045
|
+
mimeType,
|
|
1046
|
+
data,
|
|
1047
|
+
name,
|
|
1048
|
+
extension: mimeType.split('/')[1]
|
|
1049
|
+
});
|
|
1050
|
+
imageNode = {
|
|
1051
|
+
type: 'image',
|
|
1052
|
+
metadata: {
|
|
1053
|
+
attachmentName: name,
|
|
1054
|
+
altText: alt,
|
|
1055
|
+
title: node.attributes?.title,
|
|
1056
|
+
width,
|
|
1057
|
+
align
|
|
1058
|
+
}
|
|
1059
|
+
};
|
|
1060
|
+
}
|
|
1061
|
+
else {
|
|
1062
|
+
imageNode = {
|
|
1063
|
+
type: 'image',
|
|
1064
|
+
metadata: {
|
|
1065
|
+
url: src,
|
|
1066
|
+
altText: alt,
|
|
1067
|
+
title: node.attributes?.title,
|
|
1068
|
+
width,
|
|
1069
|
+
align
|
|
1070
|
+
}
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
else {
|
|
1075
|
+
imageNode = {
|
|
1076
|
+
type: 'image',
|
|
1077
|
+
metadata: {
|
|
1078
|
+
url: src,
|
|
1079
|
+
altText: alt,
|
|
1080
|
+
title: node.attributes?.title,
|
|
1081
|
+
anchorIds: anchorIds.length > 0 ? anchorIds : undefined,
|
|
1082
|
+
width,
|
|
1083
|
+
align
|
|
1084
|
+
}
|
|
1085
|
+
};
|
|
1086
|
+
}
|
|
1087
|
+
if (config.includeRawContent) {
|
|
1088
|
+
imageNode.rawContent = '<img>';
|
|
1089
|
+
}
|
|
1090
|
+
return imageNode;
|
|
1091
|
+
}
|
|
1092
|
+
if (tagName === 'a') {
|
|
1093
|
+
const href = node.attributes?.href;
|
|
1094
|
+
const wikilinkPage = node.attributes?.['data-wikilink-page'];
|
|
1095
|
+
const children = parseChildren(node, newFormatting, listContext);
|
|
1096
|
+
if (wikilinkPage !== undefined) {
|
|
1097
|
+
children.forEach(c => {
|
|
1098
|
+
if (c.type === 'text') {
|
|
1099
|
+
c.metadata = { ...c.metadata, link: wikilinkPage, linkType: 'internal', wikilink: true };
|
|
1100
|
+
}
|
|
1101
|
+
});
|
|
1102
|
+
}
|
|
1103
|
+
else if (node.attributes?.['data-wikilink'] !== undefined) {
|
|
1104
|
+
// Attribute-driven wikilink shape: the page lives in data-target, the display
|
|
1105
|
+
// text is the anchor's own content (or data-alias/data-target when the anchor
|
|
1106
|
+
// is empty). data-wikilink-page above keeps precedence over this form.
|
|
1107
|
+
const page = decodeEntities(node.attributes['data-target'] || '');
|
|
1108
|
+
if (!children.some(c => c.type === 'text')) {
|
|
1109
|
+
children.push({
|
|
1110
|
+
type: 'text',
|
|
1111
|
+
text: decodeEntities(node.attributes['data-alias'] || node.attributes['data-target'] || ''),
|
|
1112
|
+
});
|
|
1113
|
+
}
|
|
1114
|
+
children.forEach(c => {
|
|
1115
|
+
if (c.type === 'text') {
|
|
1116
|
+
c.metadata = { ...c.metadata, link: page, linkType: 'internal', wikilink: true };
|
|
1117
|
+
}
|
|
1118
|
+
});
|
|
1119
|
+
}
|
|
1120
|
+
else if (href) {
|
|
1121
|
+
const linkType = href.startsWith('#') ? 'internal' : 'external';
|
|
1122
|
+
const linkTitle = node.attributes?.title;
|
|
1123
|
+
children.forEach(c => {
|
|
1124
|
+
if (c.type === 'text') {
|
|
1125
|
+
c.metadata = { ...c.metadata, link: href, linkType, title: linkTitle };
|
|
1126
|
+
}
|
|
1127
|
+
});
|
|
1128
|
+
}
|
|
1129
|
+
return children;
|
|
1130
|
+
}
|
|
1131
|
+
if (tagName === 'br') {
|
|
1132
|
+
// A <br> is a hard line break: `carriageReturn` so the Markdown generator emits a
|
|
1133
|
+
// hard break (` \n`, or a `<br>` inside a table cell) that re-imports as a <br>.
|
|
1134
|
+
// `textWrapping` emitted a bare `\n` in a paragraph, which re-imports as a space.
|
|
1135
|
+
const brNode = { type: 'break', metadata: { breakType: 'carriageReturn' } };
|
|
1136
|
+
if (config.includeRawContent) {
|
|
1137
|
+
brNode.rawContent = '<br/>';
|
|
1138
|
+
}
|
|
1139
|
+
return brNode;
|
|
1140
|
+
}
|
|
1141
|
+
if (tagName === 'hr') {
|
|
1142
|
+
// A horizontal rule is a thematic break. This library tags an office page break
|
|
1143
|
+
// as <hr class="page-break"> on emission, so that variant round-trips back to a
|
|
1144
|
+
// page break; every other <hr> is thematic. Previously <hr> was dropped entirely.
|
|
1145
|
+
const isPageBreak = (node.attributes?.class || '').split(/\s+/).includes('page-break');
|
|
1146
|
+
const hrNode = { type: 'break', metadata: { breakType: isPageBreak ? 'page' : 'thematic' } };
|
|
1147
|
+
if (config.includeRawContent) {
|
|
1148
|
+
hrNode.rawContent = '<hr/>';
|
|
1149
|
+
}
|
|
1150
|
+
return hrNode;
|
|
1151
|
+
}
|
|
1152
|
+
if (tagName === 'pre') {
|
|
1153
|
+
const codeNode = node.children.find(c => c.tagName === 'code');
|
|
1154
|
+
let language;
|
|
1155
|
+
let codeText = '';
|
|
1156
|
+
if (codeNode) {
|
|
1157
|
+
const classAttr = codeNode.attributes?.class || '';
|
|
1158
|
+
const langMatch = classAttr.split(' ').find((c) => c.startsWith('language-'));
|
|
1159
|
+
if (langMatch)
|
|
1160
|
+
language = langMatch.replace('language-', '');
|
|
1161
|
+
// Decode entities: the code body is stored raw, so `<`/`>`/`&` (e.g. a
|
|
1162
|
+
// mermaid `-->` arrow, or `a < b` in a snippet) must be turned back into text.
|
|
1163
|
+
codeText = decodeEntities(codeNode.children.map(c => c.text || '').join(''));
|
|
1164
|
+
}
|
|
1165
|
+
else {
|
|
1166
|
+
codeText = decodeEntities(node.children.map(c => c.text || '').join(''));
|
|
1167
|
+
}
|
|
1168
|
+
// A `mermaid` class token (on the <pre> or its <code>) names the language when no
|
|
1169
|
+
// explicit language-* class is present - some producers emit <pre class="mermaid">.
|
|
1170
|
+
if (!language && ((node.attributes?.class || '').split(/\s+/).includes('mermaid') ||
|
|
1171
|
+
(codeNode?.attributes?.class || '').split(/\s+/).includes('mermaid'))) {
|
|
1172
|
+
language = 'mermaid';
|
|
1173
|
+
}
|
|
1174
|
+
const preNode = {
|
|
1175
|
+
type: 'code',
|
|
1176
|
+
text: codeText,
|
|
1177
|
+
metadata: { language, anchorIds: anchorIds.length > 0 ? anchorIds : undefined }
|
|
1178
|
+
};
|
|
1179
|
+
if (config.includeRawContent) {
|
|
1180
|
+
preNode.rawContent = '<pre>...</pre>';
|
|
1181
|
+
}
|
|
1182
|
+
return preNode;
|
|
1183
|
+
}
|
|
1184
|
+
if (tagName === 'script' || tagName === 'style' || tagName === '!doctype') {
|
|
1185
|
+
return null;
|
|
1186
|
+
}
|
|
1187
|
+
return parseChildren(node, newFormatting, listContext);
|
|
1188
|
+
}
|
|
1189
|
+
return null;
|
|
1190
|
+
};
|
|
1191
|
+
// Extract <section data-footnotes> up front so its definitions are available to
|
|
1192
|
+
// <sup data-footnote-ref> references encountered anywhere earlier in the body.
|
|
1193
|
+
const findFootnotesSection = (n) => {
|
|
1194
|
+
if (n.tagName === 'section' && n.attributes?.['data-footnotes'] !== undefined)
|
|
1195
|
+
return n;
|
|
1196
|
+
for (const child of n.children) {
|
|
1197
|
+
const found = findFootnotesSection(child);
|
|
1198
|
+
if (found)
|
|
1199
|
+
return found;
|
|
1200
|
+
}
|
|
1201
|
+
return undefined;
|
|
1202
|
+
};
|
|
1203
|
+
const footnotesSectionNode = findFootnotesSection(body);
|
|
1204
|
+
if (footnotesSectionNode) {
|
|
1205
|
+
for (const item of footnotesSectionNode.children) {
|
|
1206
|
+
if (item.type !== 'element')
|
|
1207
|
+
continue;
|
|
1208
|
+
const key = item.attributes?.['data-footnote-id'];
|
|
1209
|
+
if (!key)
|
|
1210
|
+
continue;
|
|
1211
|
+
// Strip the generated back-reference link ("↩") - it's round-trip plumbing,
|
|
1212
|
+
// not part of the footnote's actual content.
|
|
1213
|
+
const filteredChildren = item.children.filter(c => !(c.tagName === 'a' && (c.attributes?.href || '').startsWith('#footnote-ref-')));
|
|
1214
|
+
const contentNodes = [];
|
|
1215
|
+
for (const child of filteredChildren) {
|
|
1216
|
+
const parsed = parseNode(child);
|
|
1217
|
+
if (parsed) {
|
|
1218
|
+
if (Array.isArray(parsed))
|
|
1219
|
+
contentNodes.push(...parsed);
|
|
1220
|
+
else
|
|
1221
|
+
contentNodes.push(parsed);
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
footnoteDefinitions.set(key, contentNodes);
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
for (const child of body.children) {
|
|
1228
|
+
const parsed = parseNode(child);
|
|
1229
|
+
if (parsed) {
|
|
1230
|
+
if (Array.isArray(parsed)) {
|
|
1231
|
+
parsed.forEach(p => {
|
|
1232
|
+
if (p.type === 'text') {
|
|
1233
|
+
// Wrap direct body text in paragraphs
|
|
1234
|
+
content.push({ type: 'paragraph', children: [p] });
|
|
1235
|
+
}
|
|
1236
|
+
else {
|
|
1237
|
+
content.push(p);
|
|
1238
|
+
}
|
|
1239
|
+
});
|
|
1240
|
+
}
|
|
1241
|
+
else {
|
|
1242
|
+
if (parsed.type === 'text') {
|
|
1243
|
+
content.push({ type: 'paragraph', children: [parsed] });
|
|
1244
|
+
}
|
|
1245
|
+
else {
|
|
1246
|
+
content.push(parsed);
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
// Orphan footnote definitions: a `<section data-footnotes>` entry that no `<sup
|
|
1252
|
+
// data-footnote-ref>` consumed would otherwise be dropped (it is skipped in the body walk and
|
|
1253
|
+
// only materialised via a reference). Recover them as trailing `unreferenced` note nodes, the
|
|
1254
|
+
// same shape MarkdownParser produces, so md -> html -> md preserves the definition instead of
|
|
1255
|
+
// turning it into junk text with a dead back-link.
|
|
1256
|
+
for (const [key, definition] of footnoteDefinitions) {
|
|
1257
|
+
if (referencedFootnoteKeys.has(key))
|
|
1258
|
+
continue;
|
|
1259
|
+
content.push({
|
|
1260
|
+
type: 'note',
|
|
1261
|
+
text: (definition || []).map(d => d.text || '').join(''),
|
|
1262
|
+
children: definition || [],
|
|
1263
|
+
metadata: { noteType: 'footnote', noteId: key, unreferenced: true },
|
|
1264
|
+
});
|
|
1265
|
+
}
|
|
1266
|
+
const toTextSync = () => content.map(n => {
|
|
1267
|
+
const getText = (node) => {
|
|
1268
|
+
if (node.type === 'text' || node.type === 'code')
|
|
1269
|
+
return node.text || '';
|
|
1270
|
+
if (node.type === 'break')
|
|
1271
|
+
return '\n';
|
|
1272
|
+
// Childless nodes still carry meaningful text - fall back to it instead of
|
|
1273
|
+
// silently vanishing from plain-text/RAG-chunk output.
|
|
1274
|
+
if (node.type === 'embed')
|
|
1275
|
+
return node.metadata?.url || '';
|
|
1276
|
+
if (node.children) {
|
|
1277
|
+
const isBlock = ['table', 'row', 'list', 'sheet', 'slide', 'admonition', 'definitionList'].includes(node.type);
|
|
1278
|
+
return node.children.map(getText).join(isBlock ? config.newlineDelimiter : '');
|
|
1279
|
+
}
|
|
1280
|
+
return '';
|
|
1281
|
+
};
|
|
1282
|
+
return getText(n);
|
|
1283
|
+
}).join(config.newlineDelimiter)
|
|
1284
|
+
.replace(/\n{3,}/g, '\n\n'); // Normalize excessive whitespace
|
|
1285
|
+
return (0, astUtils_js_1.createAST)('html', metadata, content, attachments, config, undefined, toTextSync);
|
|
1286
|
+
};
|
|
1287
|
+
exports.parseHtml = parseHtml;
|