@sixb/agent 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/agent-prompt.d.ts +13 -0
- package/dist/agent-prompt.d.ts.map +1 -1
- package/dist/agent-runtime/preflight.d.ts.map +1 -1
- package/dist/artifacts/layout-check/layout-check.d.ts.map +1 -1
- package/dist/background-commands.d.ts +95 -0
- package/dist/background-commands.d.ts.map +1 -0
- package/dist/background-notices.d.ts +31 -0
- package/dist/background-notices.d.ts.map +1 -0
- package/dist/computer-runner.d.ts +63 -2
- package/dist/computer-runner.d.ts.map +1 -1
- package/dist/context-compaction.d.ts +3 -0
- package/dist/context-compaction.d.ts.map +1 -1
- package/dist/execution-plan.d.ts +12 -2
- package/dist/execution-plan.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2287 -230
- package/dist/index.js.map +28 -18
- package/dist/login-shell.d.ts +16 -0
- package/dist/login-shell.d.ts.map +1 -0
- package/dist/run-child.d.ts +3 -0
- package/dist/run-child.d.ts.map +1 -1
- package/dist/run-conversation-turn.d.ts +6 -2
- package/dist/run-conversation-turn.d.ts.map +1 -1
- package/dist/run-environment.d.ts +12 -2
- package/dist/run-environment.d.ts.map +1 -1
- package/dist/run-sandbox.d.ts +23 -0
- package/dist/run-sandbox.d.ts.map +1 -1
- package/dist/sandbox-files.d.ts.map +1 -1
- package/dist/tools/background.d.ts +50 -0
- package/dist/tools/background.d.ts.map +1 -0
- package/dist/tools/bash.d.ts +12 -4
- package/dist/tools/bash.d.ts.map +1 -1
- package/dist/tools/model-spec.d.ts +9 -2
- package/dist/tools/model-spec.d.ts.map +1 -1
- package/dist/tools/tool-outputs.d.ts +12 -2
- package/dist/tools/tool-outputs.d.ts.map +1 -1
- package/dist/tools/web-fetch/cache.d.ts +33 -0
- package/dist/tools/web-fetch/cache.d.ts.map +1 -0
- package/dist/tools/web-fetch/html-markdown.d.ts +25 -0
- package/dist/tools/web-fetch/html-markdown.d.ts.map +1 -0
- package/dist/tools/web-fetch/network.d.ts +55 -0
- package/dist/tools/web-fetch/network.d.ts.map +1 -0
- package/dist/tools/web-fetch/tool.d.ts +96 -0
- package/dist/tools/web-fetch/tool.d.ts.map +1 -0
- package/dist/tools/web-search.d.ts +40 -0
- package/dist/tools/web-search.d.ts.map +1 -0
- package/dist/workspace-instructions.d.ts +15 -0
- package/dist/workspace-instructions.d.ts.map +1 -0
- package/package.json +13 -4
- package/src/agent-prompt.ts +60 -1
- package/src/agent-runtime/preflight.ts +2 -1
- package/src/artifacts/layout-check/layout-check.ts +4 -4
- package/src/background-commands.ts +573 -0
- package/src/background-notices.ts +178 -0
- package/src/computer-runner.ts +211 -9
- package/src/context-compaction.ts +4 -1
- package/src/execution-plan.ts +15 -2
- package/src/index.ts +3 -0
- package/src/login-shell.ts +18 -0
- package/src/run-child.ts +5 -0
- package/src/run-conversation-turn.ts +23 -4
- package/src/run-environment.ts +92 -12
- package/src/run-sandbox.ts +41 -0
- package/src/sandbox-files.ts +2 -1
- package/src/tools/background.ts +127 -0
- package/src/tools/bash.ts +80 -11
- package/src/tools/model-spec.ts +25 -3
- package/src/tools/tool-outputs.ts +46 -6
- package/src/tools/web-fetch/cache.ts +91 -0
- package/src/tools/web-fetch/html-markdown.ts +479 -0
- package/src/tools/web-fetch/network.ts +452 -0
- package/src/tools/web-fetch/tool.ts +619 -0
- package/src/tools/web-search.ts +139 -0
- package/src/workspace-instructions.ts +163 -0
|
@@ -0,0 +1,479 @@
|
|
|
1
|
+
import type { Element, ElementContent, Nodes, Root, RootContent } from "hast"
|
|
2
|
+
import { fromParse5 } from "hast-util-from-parse5"
|
|
3
|
+
import { toMdast } from "hast-util-to-mdast"
|
|
4
|
+
import type { Nodes as MdastNodes } from "mdast"
|
|
5
|
+
import { gfmToMarkdown } from "mdast-util-gfm"
|
|
6
|
+
import { toMarkdown } from "mdast-util-to-markdown"
|
|
7
|
+
import { type DefaultTreeAdapterMap, defaultTreeAdapter, parse, type TreeAdapter } from "parse5"
|
|
8
|
+
|
|
9
|
+
/** A fetched HTML page, as the network returned it. */
|
|
10
|
+
export interface HtmlPage {
|
|
11
|
+
readonly bytes: Uint8Array
|
|
12
|
+
/** The response's `content-type` header, which may carry the charset. Empty when absent. */
|
|
13
|
+
readonly contentType: string
|
|
14
|
+
/** The URL the page was finally read from, after redirects: relative links resolve against it. */
|
|
15
|
+
readonly url: string | URL
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** The page's readable text. */
|
|
19
|
+
export interface HtmlMarkdown {
|
|
20
|
+
/** `og:title`, else `<title>`; empty when the page names itself nowhere. */
|
|
21
|
+
readonly title: string
|
|
22
|
+
readonly markdown: string
|
|
23
|
+
/** The page was cut where it nested too deep or held too many elements: the rest is missing. */
|
|
24
|
+
readonly truncated?: true
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Elements whose content is never part of what a page says: code, chrome and widgets.
|
|
29
|
+
*
|
|
30
|
+
* `head` goes too, once the title and `<base>` are read from it.
|
|
31
|
+
*/
|
|
32
|
+
const DROPPED_ELEMENTS = new Set([
|
|
33
|
+
"aside",
|
|
34
|
+
"base",
|
|
35
|
+
"footer",
|
|
36
|
+
"form",
|
|
37
|
+
"head",
|
|
38
|
+
"header",
|
|
39
|
+
"iframe",
|
|
40
|
+
"nav",
|
|
41
|
+
"noscript",
|
|
42
|
+
"script",
|
|
43
|
+
"style",
|
|
44
|
+
"svg",
|
|
45
|
+
"template",
|
|
46
|
+
])
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* How deep the page may nest: the rest of the document is cut where it goes deeper.
|
|
50
|
+
*
|
|
51
|
+
* Unbounded, a page of nested `<div>`s costs parse5 quadratic time (40k levels take seconds) and
|
|
52
|
+
* overflows the stack of every utility after it, which recurse once per level (at ~10k levels).
|
|
53
|
+
* Below that, conversion time still grows with depth × content. Real pages stay far below this
|
|
54
|
+
* (Chromium caps its tree at 512).
|
|
55
|
+
*/
|
|
56
|
+
const MAX_DEPTH = 256
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* How many elements the page may place before the rest of it is cut, each counting one plus one
|
|
60
|
+
* per {@link DEPTH_PER_ELEMENT} levels it is nested.
|
|
61
|
+
*
|
|
62
|
+
* Conversion is synchronous and costs up to ~14 µs an element, more the deeper it sits: markdown
|
|
63
|
+
* indents every line of a nested quote or list. Bytes alone do not bound it: 2 MB of
|
|
64
|
+
* `<a><b><p>x</a>` is a million elements and seven seconds. This budget is about half a second at
|
|
65
|
+
* worst, and holds a 2 MB Wikipedia article (~25k) whole.
|
|
66
|
+
*/
|
|
67
|
+
const MAX_ELEMENTS = 40_000
|
|
68
|
+
const DEPTH_PER_ELEMENT = 32
|
|
69
|
+
|
|
70
|
+
/** The most nodes converted in one piece; see {@link slices}. */
|
|
71
|
+
const SLICE_NODES = 2_000
|
|
72
|
+
|
|
73
|
+
/** Bytes the charset `<meta>` must sit within, as the HTML spec's prescan reads them. */
|
|
74
|
+
const CHARSET_PRESCAN_BYTES = 1024
|
|
75
|
+
|
|
76
|
+
type Parse5Node = DefaultTreeAdapterMap["node"]
|
|
77
|
+
type Parse5ParentNode = DefaultTreeAdapterMap["parentNode"]
|
|
78
|
+
type Parse5Document = DefaultTreeAdapterMap["document"]
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Convert a fetched HTML page to markdown, keeping its content and dropping its chrome.
|
|
82
|
+
*
|
|
83
|
+
* `parse5` builds the tree the way a browser would, whatever the markup; the page is then pruned
|
|
84
|
+
* (scripts, navigation, hidden elements), narrowed to its single `main` or `article` when it has
|
|
85
|
+
* one, and written as GFM markdown. Links and images are absolute.
|
|
86
|
+
*/
|
|
87
|
+
export function htmlToMarkdown(page: HtmlPage): HtmlMarkdown {
|
|
88
|
+
const html = decodeHtml(page.bytes, page.contentType)
|
|
89
|
+
|
|
90
|
+
const parsed = parseBounded(html)
|
|
91
|
+
const tree = fromParse5(parsed.document) as Root
|
|
92
|
+
const title = pageTitle(tree)
|
|
93
|
+
const base = baseUrl(tree, page.url)
|
|
94
|
+
|
|
95
|
+
const content = selectContent(prune(tree, base))
|
|
96
|
+
const markdown = collapseBlankLines(slices(content).map(writeMarkdown).join("\n\n"))
|
|
97
|
+
|
|
98
|
+
return { title, markdown, ...(parsed.truncated ? { truncated: true } : {}) }
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function writeMarkdown(slice: Root): string {
|
|
102
|
+
const tree = toMdast(slice)
|
|
103
|
+
unwrapEmptyInline(tree)
|
|
104
|
+
|
|
105
|
+
return toMarkdown(tree, {
|
|
106
|
+
bullet: "-",
|
|
107
|
+
emphasis: "_",
|
|
108
|
+
fences: true,
|
|
109
|
+
listItemIndent: "one",
|
|
110
|
+
rule: "-",
|
|
111
|
+
extensions: [gfmToMarkdown()],
|
|
112
|
+
})
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Inline nodes written as markers around their content. */
|
|
116
|
+
const WRAPPING_INLINE = new Set(["delete", "emphasis", "strong", "link"])
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Replace inline nodes that show nothing by what they hold: written, an empty or blank one is its
|
|
120
|
+
* markers alone (`~~~~` for `<del></del>`), which at a line's start opens a code fence. Blank
|
|
121
|
+
* inline code keeps its spaces as text.
|
|
122
|
+
*/
|
|
123
|
+
function unwrapEmptyInline(node: MdastNodes): void {
|
|
124
|
+
if (!("children" in node)) return
|
|
125
|
+
|
|
126
|
+
const children: MdastNodes[] = []
|
|
127
|
+
for (const child of node.children) {
|
|
128
|
+
unwrapEmptyInline(child)
|
|
129
|
+
|
|
130
|
+
if (child.type === "inlineCode" && !child.value.trim()) {
|
|
131
|
+
children.push({ type: "text", value: child.value })
|
|
132
|
+
} else if (WRAPPING_INLINE.has(child.type) && "children" in child && isBlank(child.children)) {
|
|
133
|
+
children.push(...child.children)
|
|
134
|
+
} else {
|
|
135
|
+
children.push(child)
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
node.children = children as typeof node.children
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Only whitespace text, or nothing. */
|
|
143
|
+
function isBlank(nodes: readonly MdastNodes[]): boolean {
|
|
144
|
+
return nodes.every((node) => node.type === "text" && !node.value.trim())
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Decode with the charset the header names, else the one a `<meta>` names near the top, else
|
|
149
|
+
* UTF-8. A byte-order mark wins over both, as in a browser. An unknown label decodes as UTF-8.
|
|
150
|
+
*/
|
|
151
|
+
function decodeHtml(bytes: Uint8Array, contentType: string): string {
|
|
152
|
+
const label = bomCharset(bytes) ?? headerCharset(contentType) ?? metaCharset(bytes) ?? "utf-8"
|
|
153
|
+
|
|
154
|
+
return decoderFor(label).decode(bytes)
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function bomCharset(bytes: Uint8Array): string | undefined {
|
|
158
|
+
if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) return "utf-8"
|
|
159
|
+
if (bytes[0] === 0xfe && bytes[1] === 0xff) return "utf-16be"
|
|
160
|
+
if (bytes[0] === 0xff && bytes[1] === 0xfe) return "utf-16le"
|
|
161
|
+
|
|
162
|
+
return undefined
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function headerCharset(contentType: string): string | undefined {
|
|
166
|
+
return /charset\s*=\s*["']?([^"';\s]+)/i.exec(contentType)?.[1]
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** `<meta charset="…">` or `<meta http-equiv="content-type" content="…; charset=…">`. */
|
|
170
|
+
function metaCharset(bytes: Uint8Array): string | undefined {
|
|
171
|
+
// Latin-1 maps every byte to one character, so ASCII markup reads the same in any charset.
|
|
172
|
+
const head = new TextDecoder("latin1").decode(bytes.subarray(0, CHARSET_PRESCAN_BYTES))
|
|
173
|
+
|
|
174
|
+
return /<meta[^>]+charset\s*=\s*["']?([^"'>;\s/]+)/i.exec(head)?.[1]
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function decoderFor(label: string): TextDecoder {
|
|
178
|
+
try {
|
|
179
|
+
return new TextDecoder(label.trim())
|
|
180
|
+
} catch {
|
|
181
|
+
return new TextDecoder("utf-8")
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Parse like a browser, but stop the document where it nests deeper than {@link MAX_DEPTH} or
|
|
187
|
+
* places more than {@link MAX_ELEMENTS} elements.
|
|
188
|
+
*
|
|
189
|
+
* Throwing from the tree adapter is the only way to bound parse5: it keeps every open element on
|
|
190
|
+
* a stack and scans that stack for most start tags, so its time grows with depth × tags.
|
|
191
|
+
*/
|
|
192
|
+
function parseBounded(html: string): { document: Parse5Document; truncated: boolean } {
|
|
193
|
+
const depths = new WeakMap<Parse5Node, number>()
|
|
194
|
+
let document: Parse5Document | undefined
|
|
195
|
+
let elements = 0
|
|
196
|
+
|
|
197
|
+
const place = (parent: Parse5ParentNode, child: Parse5Node): void => {
|
|
198
|
+
if (!defaultTreeAdapter.isElementNode(child)) return
|
|
199
|
+
|
|
200
|
+
const depth = (depths.get(parent) ?? 0) + 1
|
|
201
|
+
if (depth > MAX_DEPTH) throw new Cut()
|
|
202
|
+
|
|
203
|
+
elements += 1 + depth / DEPTH_PER_ELEMENT
|
|
204
|
+
if (elements > MAX_ELEMENTS) throw new Cut()
|
|
205
|
+
|
|
206
|
+
depths.set(child, depth)
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const treeAdapter: TreeAdapter<DefaultTreeAdapterMap> = {
|
|
210
|
+
...defaultTreeAdapter,
|
|
211
|
+
|
|
212
|
+
createDocument() {
|
|
213
|
+
document = defaultTreeAdapter.createDocument()
|
|
214
|
+
return document
|
|
215
|
+
},
|
|
216
|
+
|
|
217
|
+
appendChild(parent, child) {
|
|
218
|
+
place(parent, child)
|
|
219
|
+
defaultTreeAdapter.appendChild(parent, child)
|
|
220
|
+
},
|
|
221
|
+
|
|
222
|
+
insertBefore(parent, child, reference) {
|
|
223
|
+
place(parent, child)
|
|
224
|
+
defaultTreeAdapter.insertBefore(parent, child, reference)
|
|
225
|
+
},
|
|
226
|
+
|
|
227
|
+
// A template's content is a fragment of its own; it starts at the template's depth.
|
|
228
|
+
setTemplateContent(template, content) {
|
|
229
|
+
depths.set(content, depths.get(template) ?? 0)
|
|
230
|
+
defaultTreeAdapter.setTemplateContent(template, content)
|
|
231
|
+
},
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
try {
|
|
235
|
+
return { document: parse(html, { treeAdapter }), truncated: false }
|
|
236
|
+
} catch (error) {
|
|
237
|
+
if (error instanceof Cut && document) return { document, truncated: true }
|
|
238
|
+
throw error
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
class Cut extends Error {}
|
|
243
|
+
|
|
244
|
+
/** `og:title`, else `<title>`, both from the head. */
|
|
245
|
+
function pageTitle(tree: Root): string {
|
|
246
|
+
const head = findElement(tree, "head")
|
|
247
|
+
if (!head) return ""
|
|
248
|
+
|
|
249
|
+
for (const node of head.children) {
|
|
250
|
+
if (!isElement(node, "meta")) continue
|
|
251
|
+
|
|
252
|
+
const property = node.properties.property ?? node.properties.name
|
|
253
|
+
const content = node.properties.content
|
|
254
|
+
if (property === "og:title" && typeof content === "string" && content.trim()) {
|
|
255
|
+
return content.trim()
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
const title = findElement(head, "title")
|
|
260
|
+
if (!title) return ""
|
|
261
|
+
|
|
262
|
+
return title.children
|
|
263
|
+
.map((child) => (child.type === "text" ? child.value : ""))
|
|
264
|
+
.join("")
|
|
265
|
+
.replace(/\s+/g, " ")
|
|
266
|
+
.trim()
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** The page's URL, or its `<base href>` resolved against it. */
|
|
270
|
+
function baseUrl(tree: Root, pageUrl: string | URL): URL | undefined {
|
|
271
|
+
const page = parseUrl(String(pageUrl))
|
|
272
|
+
if (!page) return undefined
|
|
273
|
+
|
|
274
|
+
const base = findElement(tree, "base")
|
|
275
|
+
const href = base?.properties.href
|
|
276
|
+
|
|
277
|
+
return (typeof href === "string" && parseUrl(href, page)) || page
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Drop what is not content, and make every link and image absolute.
|
|
282
|
+
*
|
|
283
|
+
* A link that goes nowhere a reader can follow (`javascript:`, malformed) keeps its text; an
|
|
284
|
+
* image that is not on the web (`data:`, malformed) is dropped, since its bytes would only bloat
|
|
285
|
+
* the page.
|
|
286
|
+
*/
|
|
287
|
+
function prune<T extends Root | Element>(node: T, base: URL | undefined): T {
|
|
288
|
+
const children: ElementContent[] = []
|
|
289
|
+
|
|
290
|
+
for (const child of node.children) {
|
|
291
|
+
if (child.type === "comment" || child.type === "doctype") continue
|
|
292
|
+
|
|
293
|
+
if (child.type !== "element") {
|
|
294
|
+
children.push(child)
|
|
295
|
+
continue
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
if (isDropped(child)) continue
|
|
299
|
+
|
|
300
|
+
if (child.tagName === "a" && typeof child.properties.href === "string") {
|
|
301
|
+
const href = webUrl(child.properties.href, base, ["http:", "https:", "mailto:"])
|
|
302
|
+
|
|
303
|
+
if (!href) {
|
|
304
|
+
for (const grandchild of prune(child, base).children) children.push(grandchild)
|
|
305
|
+
continue
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
child.properties.href = href
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
if (child.tagName === "img") {
|
|
312
|
+
const src = webUrl(String(child.properties.src ?? ""), base, ["http:", "https:"])
|
|
313
|
+
if (!src) continue
|
|
314
|
+
|
|
315
|
+
child.properties.src = src
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
children.push(prune(child, base))
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
node.children = children as T["children"]
|
|
322
|
+
return node
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function isDropped(element: Element): boolean {
|
|
326
|
+
if (DROPPED_ELEMENTS.has(element.tagName)) return true
|
|
327
|
+
if (element.properties.hidden) return true
|
|
328
|
+
|
|
329
|
+
return String(element.properties.ariaHidden).toLowerCase() === "true"
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** The page's single `main`, else its single `article`, else the whole page. */
|
|
333
|
+
function selectContent(tree: Root): Root {
|
|
334
|
+
for (const tagName of ["main", "article"]) {
|
|
335
|
+
const found = findElements(tree, tagName)
|
|
336
|
+
if (found.length === 1) return { type: "root", children: found }
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
return tree
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Cut the content into trees of at most {@link SLICE_NODES} nodes, converted one at a time.
|
|
344
|
+
*
|
|
345
|
+
* The syntax-tree utilities find a node's index with `indexOf` on its parent's children, so a
|
|
346
|
+
* parent with n children costs n²: 160k table rows take 25 s in one piece. A small page is one
|
|
347
|
+
* slice and converts unchanged; on a large one, an element too big for a slice is repeated
|
|
348
|
+
* around each part of its children — a long table becomes several tables, a long list several
|
|
349
|
+
* lists.
|
|
350
|
+
*/
|
|
351
|
+
function slices(content: Root): Root[] {
|
|
352
|
+
return split(content, new WeakMap())
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function split<T extends Root | Element>(node: T, sizes: WeakMap<Nodes, number>): T[] {
|
|
356
|
+
if (sizeOf(node, sizes) <= SLICE_NODES) return [node]
|
|
357
|
+
|
|
358
|
+
const parts: T[] = []
|
|
359
|
+
let group: Array<T["children"][number]> = []
|
|
360
|
+
let groupSize = 0
|
|
361
|
+
|
|
362
|
+
const flush = (): void => {
|
|
363
|
+
if (group.length === 0) return
|
|
364
|
+
|
|
365
|
+
const part = { ...node, children: group }
|
|
366
|
+
sizes.set(part, groupSize + 1)
|
|
367
|
+
parts.push(part)
|
|
368
|
+
|
|
369
|
+
group = []
|
|
370
|
+
groupSize = 0
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
for (const child of node.children) {
|
|
374
|
+
const pieces = child.type === "element" ? split(child, sizes) : [child]
|
|
375
|
+
|
|
376
|
+
for (const piece of pieces) {
|
|
377
|
+
const size = sizeOf(piece, sizes)
|
|
378
|
+
if (groupSize + size > SLICE_NODES) flush()
|
|
379
|
+
|
|
380
|
+
group.push(piece)
|
|
381
|
+
groupSize += size
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
flush()
|
|
386
|
+
return parts
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/** Nodes in the subtree, counted once and remembered. */
|
|
390
|
+
function sizeOf(node: Nodes, sizes: WeakMap<Nodes, number>): number {
|
|
391
|
+
const known = sizes.get(node)
|
|
392
|
+
if (known !== undefined) return known
|
|
393
|
+
|
|
394
|
+
let size = 1
|
|
395
|
+
if ("children" in node) {
|
|
396
|
+
for (const child of node.children) size += sizeOf(child, sizes)
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
sizes.set(node, size)
|
|
400
|
+
return size
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* Collapse runs of blank lines to one and trim, leaving fenced code exactly as written: its blank
|
|
405
|
+
* lines are part of the code.
|
|
406
|
+
*/
|
|
407
|
+
function collapseBlankLines(markdown: string): string {
|
|
408
|
+
const lines: string[] = []
|
|
409
|
+
let fence: string | undefined
|
|
410
|
+
let blank = false
|
|
411
|
+
|
|
412
|
+
for (const line of markdown.split("\n")) {
|
|
413
|
+
const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1]
|
|
414
|
+
|
|
415
|
+
if (fence) {
|
|
416
|
+
if (marker?.startsWith(fence) && line.trim() === marker) fence = undefined
|
|
417
|
+
lines.push(line)
|
|
418
|
+
continue
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
if (marker) fence = marker
|
|
422
|
+
|
|
423
|
+
const isBlank = line.trim() === ""
|
|
424
|
+
if (isBlank && blank) continue
|
|
425
|
+
|
|
426
|
+
blank = isBlank
|
|
427
|
+
lines.push(isBlank ? "" : line)
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
return lines.join("\n").trim()
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
function webUrl(value: string, base: URL | undefined, protocols: string[]): string | undefined {
|
|
434
|
+
const url = parseUrl(value.trim(), base)
|
|
435
|
+
if (!url || !protocols.includes(url.protocol)) return undefined
|
|
436
|
+
|
|
437
|
+
return url.href
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
function parseUrl(value: string, base?: URL): URL | undefined {
|
|
441
|
+
try {
|
|
442
|
+
return new URL(value, base)
|
|
443
|
+
} catch {
|
|
444
|
+
return undefined
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
function isElement(node: RootContent | ElementContent, tagName: string): node is Element {
|
|
449
|
+
return node.type === "element" && node.tagName === tagName
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
function findElement(node: Root | Element, tagName: string): Element | undefined {
|
|
453
|
+
return findElements(node, tagName, 1)[0]
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/** Elements named `tagName` under `node`, outermost first, without descending into a match. */
|
|
457
|
+
function findElements(node: Root | Element, tagName: string, limit = Infinity): Element[] {
|
|
458
|
+
const found: Element[] = []
|
|
459
|
+
const stack: Array<RootContent | ElementContent> = []
|
|
460
|
+
pushReversed(stack, node.children)
|
|
461
|
+
|
|
462
|
+
for (let child = stack.pop(); child && found.length < limit; child = stack.pop()) {
|
|
463
|
+
if (child.type !== "element") continue
|
|
464
|
+
|
|
465
|
+
if (child.tagName === tagName) {
|
|
466
|
+
found.push(child)
|
|
467
|
+
continue
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
pushReversed(stack, child.children)
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
return found
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/** Push so that popping visits `items` in document order. No spread: a node may have millions. */
|
|
477
|
+
function pushReversed<T>(stack: T[], items: readonly T[]): void {
|
|
478
|
+
for (let index = items.length - 1; index >= 0; index--) stack.push(items[index] as T)
|
|
479
|
+
}
|