@jini-ai/platform 0.5.2 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/asset-cache.d.ts +12 -1
- package/dist/asset-cache.js +9 -5
- package/dist/child-environment.d.ts +18 -0
- package/dist/child-environment.js +26 -0
- package/dist/fs/boot-identity.d.ts +10 -0
- package/dist/fs/boot-identity.js +34 -0
- package/dist/fs/directory-home.d.ts +15 -0
- package/dist/fs/directory-home.js +17 -0
- package/dist/fs/directory-registry.d.ts +50 -0
- package/dist/fs/directory-registry.js +99 -0
- package/dist/fs/directory-running-lock.d.ts +43 -0
- package/dist/fs/directory-running-lock.js +66 -0
- package/dist/fs/directory-sync.d.ts +20 -0
- package/dist/fs/directory-sync.js +87 -0
- package/dist/fs/directory-user-config.d.ts +79 -0
- package/dist/fs/directory-user-config.js +183 -0
- package/dist/fs/directory-workspace.d.ts +132 -0
- package/dist/fs/directory-workspace.js +266 -0
- package/dist/fs/durable-json.d.ts +21 -0
- package/dist/fs/durable-json.js +59 -4
- package/dist/fs/file-lock.d.ts +3 -0
- package/dist/fs/file-lock.js +16 -7
- package/dist/fs/index.d.ts +6 -0
- package/dist/fs/index.js +6 -0
- package/dist/fs/user-data-directories.d.ts +28 -0
- package/dist/fs/user-data-directories.js +53 -0
- package/dist/fs/verified-directory-move.d.ts +37 -0
- package/dist/fs/verified-directory-move.js +138 -0
- package/dist/fs/verified-file-relocation.d.ts +27 -0
- package/dist/fs/verified-file-relocation.js +115 -0
- package/dist/fs/workspace-directory.d.ts +19 -0
- package/dist/fs/workspace-directory.js +23 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/secrets/credential-sets/index.d.ts +5 -0
- package/dist/secrets/credential-sets/index.js +5 -0
- package/dist/secrets/credential-sets/vault.aad.d.ts +4 -0
- package/dist/secrets/credential-sets/vault.aad.js +8 -0
- package/dist/secrets/credential-sets/vault.meta.d.ts +10 -0
- package/dist/secrets/credential-sets/vault.meta.js +52 -0
- package/dist/secrets/credential-sets/vault.repo.memory.d.ts +51 -0
- package/dist/secrets/credential-sets/vault.repo.memory.js +148 -0
- package/dist/secrets/credential-sets/vault.service.d.ts +130 -0
- package/dist/secrets/credential-sets/vault.service.js +310 -0
- package/dist/secrets/credential-sets/vault.types.d.ts +150 -0
- package/dist/secrets/credential-sets/vault.types.js +28 -0
- package/dist/web/html-document.d.ts +70 -0
- package/dist/web/html-document.js +268 -0
- package/dist/web/html-resource.d.ts +15 -0
- package/dist/web/html-resource.js +12 -0
- package/dist/web/html-to-markdown.d.ts +11 -0
- package/dist/web/html-to-markdown.js +245 -0
- package/dist/web/index.d.ts +5 -0
- package/dist/web/index.js +5 -0
- package/dist/web/web-page.d.ts +157 -0
- package/dist/web/web-page.js +310 -0
- package/package.json +14 -3
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
import { parse, serialize } from "parse5";
|
|
2
|
+
export const MAX_LINKS = 500;
|
|
3
|
+
export const MAX_IMAGES = 300;
|
|
4
|
+
const MAX_STYLESHEETS = 100;
|
|
5
|
+
const MAX_META = 60;
|
|
6
|
+
const MAX_LINK_TEXT = 200;
|
|
7
|
+
const MAX_META_VALUE = 500;
|
|
8
|
+
export const LINK_PROTOCOLS = new Set(["http:", "https:", "mailto:", "tel:"]);
|
|
9
|
+
export const ASSET_PROTOCOLS = new Set(["http:", "https:"]);
|
|
10
|
+
/** Never visible and never page content; removed from the text and the returned HTML copy. */
|
|
11
|
+
const STRIPPED = new Set(["script", "noscript", "template"]);
|
|
12
|
+
/** Not visible text either, but `style` stays in the HTML copy because theming needs it. */
|
|
13
|
+
const INVISIBLE = new Set([...STRIPPED, "style", "head", "svg", "iframe", "object", "canvas"]);
|
|
14
|
+
const BLOCKS = new Set(["address", "article", "aside", "blockquote", "br", "dd", "div", "dl", "dt", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "ol", "p", "pre", "section", "table", "td", "th", "tr", "ul"]);
|
|
15
|
+
/** `meta name=` keys worth returning; every `og:*` property is returned as well. */
|
|
16
|
+
const META_NAMES = new Set(["generator", "theme-color", "author", "keywords", "twitter:title", "twitter:description", "twitter:image", "application-name"]);
|
|
17
|
+
export function attr({ element, name }, _optional = {}) {
|
|
18
|
+
return element.attrs.find(a => a.name === name)?.value;
|
|
19
|
+
}
|
|
20
|
+
function isElement(node) {
|
|
21
|
+
return "tagName" in node;
|
|
22
|
+
}
|
|
23
|
+
/** Iterative pre-order walk: deeply nested hostile HTML cannot overflow the stack. O(nodes). */
|
|
24
|
+
function* walk(root) {
|
|
25
|
+
const pending = [root];
|
|
26
|
+
while (pending.length) {
|
|
27
|
+
const node = pending.pop();
|
|
28
|
+
yield node;
|
|
29
|
+
// Read after the yield on purpose: stripActiveHtml filters childNodes in between.
|
|
30
|
+
const children = "childNodes" in node ? node.childNodes : [];
|
|
31
|
+
for (let i = children.length - 1; i >= 0; i--)
|
|
32
|
+
pending.push(children[i]);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
function collapse(text) {
|
|
36
|
+
return text.replace(/\s+/g, " ").trim();
|
|
37
|
+
}
|
|
38
|
+
function textOf(root) {
|
|
39
|
+
const parts = [];
|
|
40
|
+
for (const node of walk(root))
|
|
41
|
+
if (node.nodeName === "#text" && "value" in node)
|
|
42
|
+
parts.push(node.value);
|
|
43
|
+
return collapse(parts.join(" "));
|
|
44
|
+
}
|
|
45
|
+
/** Resolves against `base`; drops fragment-only, unparseable and other-protocol values, and fragments. */
|
|
46
|
+
export function absolute({ raw, base, protocols }, _optional = {}) {
|
|
47
|
+
const value = raw?.trim();
|
|
48
|
+
if (!value || value.startsWith("#"))
|
|
49
|
+
return undefined;
|
|
50
|
+
let url;
|
|
51
|
+
try {
|
|
52
|
+
url = new URL(value, base);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return undefined;
|
|
56
|
+
}
|
|
57
|
+
if (!protocols.has(url.protocol))
|
|
58
|
+
return undefined;
|
|
59
|
+
if (url.protocol === "http:" || url.protocol === "https:")
|
|
60
|
+
url.hash = "";
|
|
61
|
+
return url.href;
|
|
62
|
+
}
|
|
63
|
+
function siteHost(hostname) {
|
|
64
|
+
return hostname.toLowerCase().replace(/^www\./, "");
|
|
65
|
+
}
|
|
66
|
+
function isInternal(href, pageHost) {
|
|
67
|
+
const url = new URL(href);
|
|
68
|
+
return (url.protocol === "http:" || url.protocol === "https:") && siteHost(url.hostname) === pageHost;
|
|
69
|
+
}
|
|
70
|
+
function linkText(anchor) {
|
|
71
|
+
const text = textOf(anchor) || attr({ element: anchor, name: "aria-label" }) || attr({ element: anchor, name: "title" }) || "";
|
|
72
|
+
if (text)
|
|
73
|
+
return collapse(text).slice(0, MAX_LINK_TEXT);
|
|
74
|
+
for (const node of walk(anchor))
|
|
75
|
+
if (isElement(node) && node.tagName === "img")
|
|
76
|
+
return collapse(attr({ element: node, name: "alt" }) ?? "").slice(0, MAX_LINK_TEXT);
|
|
77
|
+
return "";
|
|
78
|
+
}
|
|
79
|
+
/** First usable image URL: lazy loaders park the real one in `data-src` behind a `data:` placeholder. */
|
|
80
|
+
export function imageSource({ img }, _optional = {}) {
|
|
81
|
+
const src = attr({ element: img, name: "src" });
|
|
82
|
+
if (src && !src.trim().startsWith("data:"))
|
|
83
|
+
return src;
|
|
84
|
+
return attr({ element: img, name: "data-src" }) ?? attr({ element: img, name: "data-lazy-src" }) ?? attr({ element: img, name: "srcset" })?.trim().split(/\s+/)[0];
|
|
85
|
+
}
|
|
86
|
+
function collect(document) {
|
|
87
|
+
const raw = { anchors: [], images: [], linkTags: [], metas: [] };
|
|
88
|
+
for (const node of walk(document)) {
|
|
89
|
+
if (!isElement(node))
|
|
90
|
+
continue;
|
|
91
|
+
switch (node.tagName) {
|
|
92
|
+
case "a":
|
|
93
|
+
raw.anchors.push(node);
|
|
94
|
+
break;
|
|
95
|
+
case "img":
|
|
96
|
+
raw.images.push(node);
|
|
97
|
+
break;
|
|
98
|
+
case "link":
|
|
99
|
+
raw.linkTags.push(node);
|
|
100
|
+
break;
|
|
101
|
+
case "meta":
|
|
102
|
+
raw.metas.push(node);
|
|
103
|
+
break;
|
|
104
|
+
case "title":
|
|
105
|
+
raw.title ??= node;
|
|
106
|
+
break;
|
|
107
|
+
case "html":
|
|
108
|
+
raw.html ??= node;
|
|
109
|
+
break;
|
|
110
|
+
case "base":
|
|
111
|
+
raw.base ??= attr({ element: node, name: "href" });
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return raw;
|
|
116
|
+
}
|
|
117
|
+
function readMeta(raw, base) {
|
|
118
|
+
const meta = {};
|
|
119
|
+
const put = (key, value) => {
|
|
120
|
+
if (value === undefined || key in meta || Object.keys(meta).length >= MAX_META)
|
|
121
|
+
return;
|
|
122
|
+
meta[key] = collapse(value).slice(0, MAX_META_VALUE);
|
|
123
|
+
};
|
|
124
|
+
for (const element of raw.metas) {
|
|
125
|
+
const property = attr({ element, name: "property" })?.toLowerCase();
|
|
126
|
+
const name = attr({ element, name: "name" })?.toLowerCase();
|
|
127
|
+
const content = attr({ element, name: "content" });
|
|
128
|
+
if (property?.startsWith("og:"))
|
|
129
|
+
put(property, property === "og:image" || property === "og:url" ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content);
|
|
130
|
+
else if (name && META_NAMES.has(name))
|
|
131
|
+
put(name, name === "twitter:image" ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content);
|
|
132
|
+
}
|
|
133
|
+
for (const element of raw.linkTags) {
|
|
134
|
+
const rel = (attr({ element, name: "rel" }) ?? "").toLowerCase().split(/\s+/);
|
|
135
|
+
const href = absolute({ raw: attr({ element, name: "href" }), base, protocols: ASSET_PROTOCOLS });
|
|
136
|
+
if (rel.includes("canonical"))
|
|
137
|
+
put("canonical", href);
|
|
138
|
+
if (rel.includes("icon"))
|
|
139
|
+
put("favicon", href);
|
|
140
|
+
if (rel.includes("apple-touch-icon"))
|
|
141
|
+
put("apple-touch-icon", href);
|
|
142
|
+
}
|
|
143
|
+
return meta;
|
|
144
|
+
}
|
|
145
|
+
function readLinks(anchors, base, pageHost) {
|
|
146
|
+
const seen = new Set();
|
|
147
|
+
const links = [];
|
|
148
|
+
let truncated = false;
|
|
149
|
+
for (const anchor of anchors) {
|
|
150
|
+
const href = absolute({ raw: attr({ element: anchor, name: "href" }), base, protocols: LINK_PROTOCOLS });
|
|
151
|
+
if (!href || seen.has(href))
|
|
152
|
+
continue;
|
|
153
|
+
seen.add(href);
|
|
154
|
+
if (links.length === MAX_LINKS) {
|
|
155
|
+
truncated = true;
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
links.push({ href, text: linkText(anchor), internal: isInternal(href, pageHost) });
|
|
159
|
+
}
|
|
160
|
+
return { links, truncated };
|
|
161
|
+
}
|
|
162
|
+
function readImages(images, base) {
|
|
163
|
+
const seen = new Set();
|
|
164
|
+
const out = [];
|
|
165
|
+
let truncated = false;
|
|
166
|
+
for (const img of images) {
|
|
167
|
+
const src = absolute({ raw: imageSource({ img }), base, protocols: ASSET_PROTOCOLS });
|
|
168
|
+
if (!src || seen.has(src))
|
|
169
|
+
continue;
|
|
170
|
+
seen.add(src);
|
|
171
|
+
if (out.length === MAX_IMAGES) {
|
|
172
|
+
truncated = true;
|
|
173
|
+
break;
|
|
174
|
+
}
|
|
175
|
+
out.push({ src, alt: collapse(attr({ element: img, name: "alt" }) ?? "") });
|
|
176
|
+
}
|
|
177
|
+
return { images: out, truncated };
|
|
178
|
+
}
|
|
179
|
+
function readStylesheets(linkTags, base) {
|
|
180
|
+
const out = new Set();
|
|
181
|
+
for (const element of linkTags) {
|
|
182
|
+
const rel = (attr({ element, name: "rel" }) ?? "").toLowerCase().split(/\s+/);
|
|
183
|
+
const href = absolute({ raw: attr({ element, name: "href" }), base, protocols: ASSET_PROTOCOLS });
|
|
184
|
+
if (rel.includes("stylesheet") && href && out.size < MAX_STYLESHEETS)
|
|
185
|
+
out.add(href);
|
|
186
|
+
}
|
|
187
|
+
return [...out];
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Extracts page facts. Relative URLs resolve against `<base href>` (itself resolved against the page
|
|
191
|
+
* URL) so links match what a browser would follow.
|
|
192
|
+
* @param required.html - Decoded page source; untrusted.
|
|
193
|
+
* @param required.pageUrl - The final URL the HTML was served from (after redirects).
|
|
194
|
+
* @returns Facts with every URL absolute; links deduped (fragment-free) and capped at {@link MAX_LINKS},
|
|
195
|
+
* images at {@link MAX_IMAGES}; `internal` ignores a leading `www.`.
|
|
196
|
+
* @complexity O(nodes) plus O(anchor subtree) per anchor for its text.
|
|
197
|
+
*/
|
|
198
|
+
export function readHtmlDocument({ html, pageUrl }, _optional = {}) {
|
|
199
|
+
const document = parse(html);
|
|
200
|
+
const raw = collect(document);
|
|
201
|
+
const base = absolute({ raw: raw.base, base: pageUrl, protocols: ASSET_PROTOCOLS }) ?? pageUrl;
|
|
202
|
+
const pageHost = siteHost(new URL(pageUrl).hostname);
|
|
203
|
+
const links = readLinks(raw.anchors, base, pageHost);
|
|
204
|
+
const images = readImages(raw.images, base);
|
|
205
|
+
const description = raw.metas.find(m => attr({ element: m, name: "name" })?.toLowerCase() === "description");
|
|
206
|
+
const title = raw.title ? textOf(raw.title) : "";
|
|
207
|
+
const lang = raw.html ? attr({ element: raw.html, name: "lang" })?.trim() : undefined;
|
|
208
|
+
const descriptionText = description ? collapse(attr({ element: description, name: "content" }) ?? "") : "";
|
|
209
|
+
return {
|
|
210
|
+
...(title ? { title } : {}),
|
|
211
|
+
...(descriptionText ? { description: descriptionText } : {}),
|
|
212
|
+
...(lang ? { lang } : {}),
|
|
213
|
+
links: links.links, linksTruncated: links.truncated,
|
|
214
|
+
images: images.images, imagesTruncated: images.truncated,
|
|
215
|
+
stylesheets: readStylesheets(raw.linkTags, base),
|
|
216
|
+
meta: readMeta(raw, base),
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Visible text with one line per block element; scripts, styles, `<head>`, SVG and embeds omitted.
|
|
221
|
+
* @complexity O(nodes).
|
|
222
|
+
*/
|
|
223
|
+
export function htmlToText({ html }, _optional = {}) {
|
|
224
|
+
const pending = [parse(html)];
|
|
225
|
+
const out = [];
|
|
226
|
+
while (pending.length) {
|
|
227
|
+
const node = pending.pop();
|
|
228
|
+
if (node === "\n") {
|
|
229
|
+
out.push("\n");
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
if (isElement(node) && INVISIBLE.has(node.tagName))
|
|
233
|
+
continue;
|
|
234
|
+
if (node.nodeName === "#text" && "value" in node) {
|
|
235
|
+
out.push(node.value);
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
if (!("childNodes" in node))
|
|
239
|
+
continue;
|
|
240
|
+
const block = isElement(node) && BLOCKS.has(node.tagName);
|
|
241
|
+
if (block) {
|
|
242
|
+
out.push("\n");
|
|
243
|
+
pending.push("\n");
|
|
244
|
+
}
|
|
245
|
+
for (let i = node.childNodes.length - 1; i >= 0; i--)
|
|
246
|
+
pending.push(node.childNodes[i]);
|
|
247
|
+
}
|
|
248
|
+
return out.join("").split("\n").map(line => line.replace(/[^\S\n]+/g, " ").trim()).filter(Boolean).join("\n");
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* The page HTML with `<script>`, `<noscript>`, `<template>`, comments, `on*` handler attributes and
|
|
252
|
+
* `javascript:` attribute values removed (styles and classes kept: theming needs them). Re-serialized by parse5, so it is well-formed rather than byte-identical.
|
|
253
|
+
* @complexity O(nodes).
|
|
254
|
+
*/
|
|
255
|
+
export function stripActiveHtml({ html }, _optional = {}) {
|
|
256
|
+
const document = parse(html);
|
|
257
|
+
for (const node of walk(document)) {
|
|
258
|
+
if (!("childNodes" in node))
|
|
259
|
+
continue;
|
|
260
|
+
const parent = node;
|
|
261
|
+
parent.childNodes = parent.childNodes.filter(child => child.nodeName !== "#comment" && !(isElement(child) && STRIPPED.has(child.tagName)));
|
|
262
|
+
// Inert copy: an agent may paste this into a page, so handlers and script URLs never survive.
|
|
263
|
+
if (isElement(node))
|
|
264
|
+
node.attrs = node.attrs.filter(a => !/^on/i.test(a.name) && !/^\s*javascript:/i.test(a.value));
|
|
265
|
+
}
|
|
266
|
+
return serialize(document);
|
|
267
|
+
}
|
|
268
|
+
//# sourceMappingURL=html-document.js.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Bound synchronous HTML parsers independently of the HTTP response byte limit. */
|
|
2
|
+
export declare const MAX_HTML_CHARS = 1000000;
|
|
3
|
+
/** Return a bounded prefix without splitting UTF-16; fetch projections expose its truncation flag.
|
|
4
|
+
* The caller may supply a lower projection budget; direct conversion always has a fixed ceiling.
|
|
5
|
+
* O(1) decision plus O(cap) prefix allocation; raw body output keeps its separate transport cap.
|
|
6
|
+
*/
|
|
7
|
+
export declare function boundHtml({ html }: {
|
|
8
|
+
html: string;
|
|
9
|
+
}, { maxChars }?: {
|
|
10
|
+
maxChars?: number;
|
|
11
|
+
}): {
|
|
12
|
+
html: string;
|
|
13
|
+
truncated: boolean;
|
|
14
|
+
};
|
|
15
|
+
//# sourceMappingURL=html-resource.d.ts.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Bound synchronous HTML parsers independently of the HTTP response byte limit. */
|
|
2
|
+
export const MAX_HTML_CHARS = 1_000_000;
|
|
3
|
+
/** Return a bounded prefix without splitting UTF-16; fetch projections expose its truncation flag.
|
|
4
|
+
* The caller may supply a lower projection budget; direct conversion always has a fixed ceiling.
|
|
5
|
+
* O(1) decision plus O(cap) prefix allocation; raw body output keeps its separate transport cap.
|
|
6
|
+
*/
|
|
7
|
+
export function boundHtml({ html }, { maxChars = MAX_HTML_CHARS } = {}) {
|
|
8
|
+
const limit = Math.min(MAX_HTML_CHARS, maxChars);
|
|
9
|
+
const end = /[\uDC00-\uDFFF]/.test(html[limit] ?? "") ? limit - 1 : limit;
|
|
10
|
+
return html.length > limit ? { html: html.slice(0, end), truncated: true } : { html, truncated: false };
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=html-resource.js.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert untrusted HTML to readable Markdown with absolute, allowlisted link/image URLs.
|
|
3
|
+
* @returns Markdown without surrounding whitespace; input is capped before parsing and errors propagate.
|
|
4
|
+
* @complexity O(nodes + output size) time/space for ordinary pages; nesting is bounded at 300.
|
|
5
|
+
* @example htmlToMarkdown({ html: "<h1>Hello</h1>", pageUrl: "https://example.com/" })
|
|
6
|
+
*/
|
|
7
|
+
export declare function htmlToMarkdown({ html, pageUrl }: {
|
|
8
|
+
html: string;
|
|
9
|
+
pageUrl: string;
|
|
10
|
+
}, _optional?: Record<string, never>): string;
|
|
11
|
+
//# sourceMappingURL=html-to-markdown.d.ts.map
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import { parse } from "parse5";
|
|
2
|
+
import { fromParse5 } from "hast-util-from-parse5";
|
|
3
|
+
import { defaultHandlers, toMdast } from "hast-util-to-mdast";
|
|
4
|
+
import { toMarkdown } from "mdast-util-to-markdown";
|
|
5
|
+
import { gfmTableToMarkdown } from "mdast-util-gfm-table";
|
|
6
|
+
import { boundHtml } from "./html-resource.js";
|
|
7
|
+
import { absolute, ASSET_PROTOCOLS, attr, imageSource, LINK_PROTOCOLS } from "./html-document.js";
|
|
8
|
+
const SKIPPED = new Set(["head", "script", "style", "noscript", "template", "svg", "iframe", "object", "canvas", "select", "input", "textarea"]);
|
|
9
|
+
/** Beyond this nesting, a subtree degrades to its plain text instead of recursing further. */
|
|
10
|
+
const MAX_DEPTH = 300;
|
|
11
|
+
/** Narrow a parser node before accessing element attributes. */
|
|
12
|
+
function isElement(node) {
|
|
13
|
+
return "tagName" in node;
|
|
14
|
+
}
|
|
15
|
+
/** Iterative so a degraded, very deep subtree cannot overflow the stack either. O(subtree nodes). */
|
|
16
|
+
function plainText(root) {
|
|
17
|
+
const pending = [root];
|
|
18
|
+
const parts = [];
|
|
19
|
+
while (pending.length) {
|
|
20
|
+
const node = pending.pop();
|
|
21
|
+
if (node.nodeName === "#text" && "value" in node)
|
|
22
|
+
parts.push(node.value);
|
|
23
|
+
else if ("childNodes" in node && !(isElement(node) && SKIPPED.has(node.tagName))) {
|
|
24
|
+
for (let i = node.childNodes.length - 1; i >= 0; i--)
|
|
25
|
+
pending.push(node.childNodes[i]);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return parts.join("").replace(/\s+/g, " ");
|
|
29
|
+
}
|
|
30
|
+
/** Resolve the document base before preparation drops the invisible head. O(head nodes). */
|
|
31
|
+
function documentBase(document, pageUrl) {
|
|
32
|
+
const head = document.childNodes.find(isElement)?.childNodes.find(node => isElement(node) && node.tagName === "head");
|
|
33
|
+
const base = head && isElement(head) ? head.childNodes.find(node => isElement(node) && node.tagName === "base") : undefined;
|
|
34
|
+
return absolute({ raw: base && isElement(base) ? attr({ element: base, name: "href" }) : undefined, base: pageUrl, protocols: ASSET_PROTOCOLS }) ?? pageUrl;
|
|
35
|
+
}
|
|
36
|
+
/** Normalize reader URLs using the same owner as page facts; mutate only the freshly parsed tree. */
|
|
37
|
+
function prepareElement(element, base) {
|
|
38
|
+
const href = element.tagName === "a" ? absolute({ raw: attr({ element, name: "href" }), base, protocols: LINK_PROTOCOLS }) : undefined;
|
|
39
|
+
const src = element.tagName === "img" ? absolute({ raw: imageSource({ img: element }), base, protocols: ASSET_PROTOCOLS }) : undefined;
|
|
40
|
+
// HTML must not supply the library's data-mdast override: it can inject arbitrary Markdown nodes.
|
|
41
|
+
// The old reader ignored table spans; retaining that contract also prevents hostile span expansion.
|
|
42
|
+
element.attrs = element.attrs.filter(attribute => !attribute.name.startsWith("data-mdast") && !["colspan", "rowspan"].includes(attribute.name));
|
|
43
|
+
if (element.tagName === "table")
|
|
44
|
+
promoteFirstRow(element);
|
|
45
|
+
if (element.tagName === "a" || element.tagName === "img") {
|
|
46
|
+
element.attrs = element.attrs.filter(attribute => !["href", "src", "title"].includes(attribute.name));
|
|
47
|
+
if (href)
|
|
48
|
+
element.attrs.push({ name: "href", value: href });
|
|
49
|
+
if (src)
|
|
50
|
+
element.attrs.push({ name: "src", value: src });
|
|
51
|
+
if (element.tagName === "img") {
|
|
52
|
+
const alt = element.attrs.find(attribute => attribute.name === "alt");
|
|
53
|
+
if (alt)
|
|
54
|
+
alt.value = alt.value.replace(/\s+/g, " ").trim();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
/** The existing reader treats the first row as the header, including tables made only of td cells. */
|
|
59
|
+
function promoteFirstRow(table) {
|
|
60
|
+
const pending = [...table.childNodes].reverse();
|
|
61
|
+
while (pending.length) {
|
|
62
|
+
const node = pending.pop();
|
|
63
|
+
if (!isElement(node) || node.tagName === "table")
|
|
64
|
+
continue;
|
|
65
|
+
if (node.tagName === "tr") {
|
|
66
|
+
for (const cell of node.childNodes) {
|
|
67
|
+
if (isElement(cell) && cell.tagName === "td") {
|
|
68
|
+
cell.tagName = "th";
|
|
69
|
+
cell.nodeName = "th";
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
for (let i = node.childNodes.length - 1; i >= 0; i--)
|
|
75
|
+
pending.push(node.childNodes[i]);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/** Drop invisible subtrees and images with no usable URL before the bridge sees them. */
|
|
79
|
+
function readable(node, base) {
|
|
80
|
+
if (node.nodeName === "#comment" || node.nodeName === "#documentType")
|
|
81
|
+
return false;
|
|
82
|
+
if (!isElement(node))
|
|
83
|
+
return true;
|
|
84
|
+
if (SKIPPED.has(node.tagName))
|
|
85
|
+
return false;
|
|
86
|
+
return node.tagName !== "img" || absolute({ raw: imageSource({ img: node }), base, protocols: ASSET_PROTOCOLS }) !== undefined;
|
|
87
|
+
}
|
|
88
|
+
/** Move whitespace outside attention before serialization; empty attention is plain whitespace.
|
|
89
|
+
* Only this newly parsed tree is mutated. Nested inline wrappers retain their formatting.
|
|
90
|
+
*/
|
|
91
|
+
function attentionChildren(parent) {
|
|
92
|
+
parent.childNodes = parent.childNodes.flatMap(node => {
|
|
93
|
+
if (!isElement(node) || !["b", "strong", "i", "em"].includes(node.tagName))
|
|
94
|
+
return [node];
|
|
95
|
+
const boundary = (last) => {
|
|
96
|
+
let edge = node;
|
|
97
|
+
while (edge && isElement(edge))
|
|
98
|
+
edge = last ? edge.childNodes.at(-1) : edge.childNodes[0];
|
|
99
|
+
return edge?.nodeName === "#text" && "value" in edge ? edge : undefined;
|
|
100
|
+
};
|
|
101
|
+
const first = boundary(false);
|
|
102
|
+
const leading = first?.value.match(/^\s+/)?.[0] ?? "";
|
|
103
|
+
if (first)
|
|
104
|
+
first.value = first.value.slice(leading.length);
|
|
105
|
+
const last = boundary(true);
|
|
106
|
+
const trailing = last?.value.match(/\s+$/)?.[0] ?? "";
|
|
107
|
+
if (last && trailing)
|
|
108
|
+
last.value = last.value.slice(0, -trailing.length);
|
|
109
|
+
const text = (value) => ({ nodeName: "#text", value, parentNode: parent });
|
|
110
|
+
const descendants = [...node.childNodes];
|
|
111
|
+
let image = false;
|
|
112
|
+
while (descendants.length && !image) {
|
|
113
|
+
const child = descendants.pop();
|
|
114
|
+
if (isElement(child) && child.tagName === "img")
|
|
115
|
+
image = true;
|
|
116
|
+
else if ("childNodes" in child)
|
|
117
|
+
descendants.push(...child.childNodes);
|
|
118
|
+
}
|
|
119
|
+
// No marker pair is emitted when whitespace was the entire content.
|
|
120
|
+
return [...(leading ? [text(leading)] : []), ...(plainText(node).trim() || image ? [node] : []), ...(trailing ? [text(trailing)] : [])];
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
function hastText(root) {
|
|
124
|
+
const pending = [root];
|
|
125
|
+
const parts = [];
|
|
126
|
+
while (pending.length) {
|
|
127
|
+
const node = pending.pop();
|
|
128
|
+
if (node.type === "text")
|
|
129
|
+
parts.push(node.value ?? "");
|
|
130
|
+
else if (node.children)
|
|
131
|
+
for (let i = node.children.length - 1; i >= 0; i--)
|
|
132
|
+
pending.push(node.children[i]);
|
|
133
|
+
}
|
|
134
|
+
return parts.join("");
|
|
135
|
+
}
|
|
136
|
+
/** A prepared anchor is visible when it contains text or a usable image, even without alt text.
|
|
137
|
+
* Preparation already removed unsafe images. O(descendant nodes) time/space, with early exit.
|
|
138
|
+
*/
|
|
139
|
+
function hasVisibleAnchorContent({ node }, _options = {}) {
|
|
140
|
+
const pending = [node];
|
|
141
|
+
while (pending.length) {
|
|
142
|
+
const child = pending.pop();
|
|
143
|
+
if (child.type === "text" && child.value?.trim())
|
|
144
|
+
return true;
|
|
145
|
+
if (child.type === "element" && child.tagName === "img")
|
|
146
|
+
return true;
|
|
147
|
+
if (child.children)
|
|
148
|
+
pending.push(...child.children);
|
|
149
|
+
}
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
/** Reader attention never rewrites neighbouring letters into character references. */
|
|
153
|
+
const readerAttention = (node, _parent, state, info) => {
|
|
154
|
+
const marker = node.type === "strong" ? "**" : "*";
|
|
155
|
+
const exit = state.enter(node.type === "strong" ? "strong" : "emphasis");
|
|
156
|
+
const tracker = state.createTracker(info);
|
|
157
|
+
tracker.move(marker);
|
|
158
|
+
const value = state.containerPhrasing(node, { ...tracker.current(), before: marker, after: marker });
|
|
159
|
+
exit();
|
|
160
|
+
return marker + value + marker;
|
|
161
|
+
};
|
|
162
|
+
/**
|
|
163
|
+
* Bound depth BEFORE the recursive parse5-to-hast bridge and the library's tree walks.
|
|
164
|
+
* O(nodes) time/space; discarded deep subtrees are scanned once for their visible text.
|
|
165
|
+
* The input is a newly allocated parser tree, never caller state.
|
|
166
|
+
*/
|
|
167
|
+
function prepare(document, base) {
|
|
168
|
+
const pending = [{ node: document, depth: 0 }];
|
|
169
|
+
while (pending.length) {
|
|
170
|
+
const { node, depth, prepared } = pending.pop();
|
|
171
|
+
if (prepared) {
|
|
172
|
+
if (isElement(node))
|
|
173
|
+
attentionChildren(node);
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (isElement(node)) {
|
|
177
|
+
prepareElement(node, base);
|
|
178
|
+
if (depth > MAX_DEPTH) {
|
|
179
|
+
const text = plainText(node);
|
|
180
|
+
node.tagName = "span";
|
|
181
|
+
node.nodeName = "span";
|
|
182
|
+
node.attrs = [];
|
|
183
|
+
node.childNodes = [{ nodeName: "#text", value: text, parentNode: node }];
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
if (!("childNodes" in node))
|
|
188
|
+
continue;
|
|
189
|
+
node.childNodes = node.childNodes.filter(child => readable(child, base));
|
|
190
|
+
pending.push({ node, depth, prepared: true });
|
|
191
|
+
for (let i = node.childNodes.length - 1; i >= 0; i--)
|
|
192
|
+
pending.push({ node: node.childNodes[i], depth: depth + 1 });
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Convert untrusted HTML to readable Markdown with absolute, allowlisted link/image URLs.
|
|
197
|
+
* @returns Markdown without surrounding whitespace; input is capped before parsing and errors propagate.
|
|
198
|
+
* @complexity O(nodes + output size) time/space for ordinary pages; nesting is bounded at 300.
|
|
199
|
+
* @example htmlToMarkdown({ html: "<h1>Hello</h1>", pageUrl: "https://example.com/" })
|
|
200
|
+
*/
|
|
201
|
+
export function htmlToMarkdown({ html, pageUrl }, _optional = {}) {
|
|
202
|
+
const document = parse(boundHtml({ html }).html);
|
|
203
|
+
prepare(document, documentBase(document, pageUrl));
|
|
204
|
+
const hast = fromParse5(document);
|
|
205
|
+
const pending = [hast];
|
|
206
|
+
while (pending.length) {
|
|
207
|
+
const node = pending.pop();
|
|
208
|
+
if (node.type === "element" && node.tagName === "code")
|
|
209
|
+
node.data = { readerInlineCode: hastText(node) };
|
|
210
|
+
if (node.children)
|
|
211
|
+
pending.push(...node.children);
|
|
212
|
+
}
|
|
213
|
+
const markdown = toMdast(hast, {
|
|
214
|
+
document: true,
|
|
215
|
+
handlers: {
|
|
216
|
+
a: (state, node) => node.properties.href && hasVisibleAnchorContent({ node }) ? defaultHandlers.a(state, node) : state.all(node),
|
|
217
|
+
code: (state, node) => {
|
|
218
|
+
const result = { type: "inlineCode", value: String(node.data?.readerInlineCode ?? "") };
|
|
219
|
+
state.patch(node, result);
|
|
220
|
+
return result;
|
|
221
|
+
},
|
|
222
|
+
// Strikethrough was plain text in this reader; no extra GFM serialization extension is needed.
|
|
223
|
+
del: (state, node) => state.all(node),
|
|
224
|
+
s: (state, node) => state.all(node),
|
|
225
|
+
strike: (state, node) => state.all(node),
|
|
226
|
+
audio: (state, node) => state.all(node),
|
|
227
|
+
video: (state, node) => state.all(node),
|
|
228
|
+
},
|
|
229
|
+
});
|
|
230
|
+
// Preparation moves whitespace out of emphasis; handlers preserve raw reader text.
|
|
231
|
+
// A nested list is indented by its parent item, including every continuation line.
|
|
232
|
+
// The serializer owns blank-line runs and inline spaces; code fences/list indents stay verbatim.
|
|
233
|
+
return toMarkdown(markdown, {
|
|
234
|
+
emphasis: "*", bullet: "-", listItemIndent: "one", fences: true,
|
|
235
|
+
rule: "-", ruleRepetition: 3, ruleSpaces: false,
|
|
236
|
+
extensions: [gfmTableToMarkdown({ tableCellPadding: true, tablePipeAlign: false }), {
|
|
237
|
+
handlers: {
|
|
238
|
+
// The reader never escaped ordinary Markdown punctuation; only table pipes need escaping.
|
|
239
|
+
text: (node, _parent, state) => state.stack.includes("tableCell") ? node.value.replace(/\|/g, "\\|") : node.value,
|
|
240
|
+
strong: readerAttention, emphasis: readerAttention,
|
|
241
|
+
},
|
|
242
|
+
}],
|
|
243
|
+
}).trim();
|
|
244
|
+
}
|
|
245
|
+
//# sourceMappingURL=html-to-markdown.js.map
|