@sixb/agent 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +2 -1
  2. package/dist/agent-prompt.d.ts +13 -0
  3. package/dist/agent-prompt.d.ts.map +1 -1
  4. package/dist/agent-runtime/preflight.d.ts.map +1 -1
  5. package/dist/artifacts/layout-check/layout-check.d.ts.map +1 -1
  6. package/dist/background-commands.d.ts +95 -0
  7. package/dist/background-commands.d.ts.map +1 -0
  8. package/dist/background-notices.d.ts +31 -0
  9. package/dist/background-notices.d.ts.map +1 -0
  10. package/dist/computer-runner.d.ts +63 -2
  11. package/dist/computer-runner.d.ts.map +1 -1
  12. package/dist/context-compaction.d.ts +3 -0
  13. package/dist/context-compaction.d.ts.map +1 -1
  14. package/dist/execution-plan.d.ts +12 -2
  15. package/dist/execution-plan.d.ts.map +1 -1
  16. package/dist/index.d.ts +2 -1
  17. package/dist/index.d.ts.map +1 -1
  18. package/dist/index.js +2287 -230
  19. package/dist/index.js.map +28 -18
  20. package/dist/login-shell.d.ts +16 -0
  21. package/dist/login-shell.d.ts.map +1 -0
  22. package/dist/run-child.d.ts +3 -0
  23. package/dist/run-child.d.ts.map +1 -1
  24. package/dist/run-conversation-turn.d.ts +6 -2
  25. package/dist/run-conversation-turn.d.ts.map +1 -1
  26. package/dist/run-environment.d.ts +12 -2
  27. package/dist/run-environment.d.ts.map +1 -1
  28. package/dist/run-sandbox.d.ts +23 -0
  29. package/dist/run-sandbox.d.ts.map +1 -1
  30. package/dist/sandbox-files.d.ts.map +1 -1
  31. package/dist/tools/background.d.ts +50 -0
  32. package/dist/tools/background.d.ts.map +1 -0
  33. package/dist/tools/bash.d.ts +12 -4
  34. package/dist/tools/bash.d.ts.map +1 -1
  35. package/dist/tools/model-spec.d.ts +9 -2
  36. package/dist/tools/model-spec.d.ts.map +1 -1
  37. package/dist/tools/tool-outputs.d.ts +12 -2
  38. package/dist/tools/tool-outputs.d.ts.map +1 -1
  39. package/dist/tools/web-fetch/cache.d.ts +33 -0
  40. package/dist/tools/web-fetch/cache.d.ts.map +1 -0
  41. package/dist/tools/web-fetch/html-markdown.d.ts +25 -0
  42. package/dist/tools/web-fetch/html-markdown.d.ts.map +1 -0
  43. package/dist/tools/web-fetch/network.d.ts +55 -0
  44. package/dist/tools/web-fetch/network.d.ts.map +1 -0
  45. package/dist/tools/web-fetch/tool.d.ts +96 -0
  46. package/dist/tools/web-fetch/tool.d.ts.map +1 -0
  47. package/dist/tools/web-search.d.ts +40 -0
  48. package/dist/tools/web-search.d.ts.map +1 -0
  49. package/dist/workspace-instructions.d.ts +15 -0
  50. package/dist/workspace-instructions.d.ts.map +1 -0
  51. package/package.json +13 -4
  52. package/src/agent-prompt.ts +60 -1
  53. package/src/agent-runtime/preflight.ts +2 -1
  54. package/src/artifacts/layout-check/layout-check.ts +4 -4
  55. package/src/background-commands.ts +573 -0
  56. package/src/background-notices.ts +178 -0
  57. package/src/computer-runner.ts +211 -9
  58. package/src/context-compaction.ts +4 -1
  59. package/src/execution-plan.ts +15 -2
  60. package/src/index.ts +3 -0
  61. package/src/login-shell.ts +18 -0
  62. package/src/run-child.ts +5 -0
  63. package/src/run-conversation-turn.ts +23 -4
  64. package/src/run-environment.ts +92 -12
  65. package/src/run-sandbox.ts +41 -0
  66. package/src/sandbox-files.ts +2 -1
  67. package/src/tools/background.ts +127 -0
  68. package/src/tools/bash.ts +80 -11
  69. package/src/tools/model-spec.ts +25 -3
  70. package/src/tools/tool-outputs.ts +46 -6
  71. package/src/tools/web-fetch/cache.ts +91 -0
  72. package/src/tools/web-fetch/html-markdown.ts +479 -0
  73. package/src/tools/web-fetch/network.ts +452 -0
  74. package/src/tools/web-fetch/tool.ts +619 -0
  75. package/src/tools/web-search.ts +139 -0
  76. package/src/workspace-instructions.ts +163 -0
@@ -0,0 +1,479 @@
1
+ import type { Element, ElementContent, Nodes, Root, RootContent } from "hast"
2
+ import { fromParse5 } from "hast-util-from-parse5"
3
+ import { toMdast } from "hast-util-to-mdast"
4
+ import type { Nodes as MdastNodes } from "mdast"
5
+ import { gfmToMarkdown } from "mdast-util-gfm"
6
+ import { toMarkdown } from "mdast-util-to-markdown"
7
+ import { type DefaultTreeAdapterMap, defaultTreeAdapter, parse, type TreeAdapter } from "parse5"
8
+
9
+ /** A fetched HTML page, as the network returned it. */
10
+ export interface HtmlPage {
11
+ readonly bytes: Uint8Array
12
+ /** The response's `content-type` header, which may carry the charset. Empty when absent. */
13
+ readonly contentType: string
14
+ /** The URL the page was finally read from, after redirects: relative links resolve against it. */
15
+ readonly url: string | URL
16
+ }
17
+
18
+ /** The page's readable text. */
19
+ export interface HtmlMarkdown {
20
+ /** `og:title`, else `<title>`; empty when the page names itself nowhere. */
21
+ readonly title: string
22
+ readonly markdown: string
23
+ /** The page was cut where it nested too deep or held too many elements: the rest is missing. */
24
+ readonly truncated?: true
25
+ }
26
+
27
+ /**
28
+ * Elements whose content is never part of what a page says: code, chrome and widgets.
29
+ *
30
+ * `head` goes too, once the title and `<base>` are read from it.
31
+ */
32
+ const DROPPED_ELEMENTS = new Set([
33
+ "aside",
34
+ "base",
35
+ "footer",
36
+ "form",
37
+ "head",
38
+ "header",
39
+ "iframe",
40
+ "nav",
41
+ "noscript",
42
+ "script",
43
+ "style",
44
+ "svg",
45
+ "template",
46
+ ])
47
+
48
+ /**
49
+ * How deep the page may nest: the rest of the document is cut where it goes deeper.
50
+ *
51
+ * Unbounded, a page of nested `<div>`s costs parse5 quadratic time (40k levels take seconds) and
52
+ * overflows the stack of every utility after it, which recurse once per level (at ~10k levels).
53
+ * Below that, conversion time still grows with depth × content. Real pages stay far below this
54
+ * (Chromium caps its tree at 512).
55
+ */
56
+ const MAX_DEPTH = 256
57
+
58
+ /**
59
+ * How many elements the page may place before the rest of it is cut, each counting one plus one
60
+ * per {@link DEPTH_PER_ELEMENT} levels it is nested.
61
+ *
62
+ * Conversion is synchronous and costs up to ~14 µs an element, more the deeper it sits: markdown
63
+ * indents every line of a nested quote or list. Bytes alone do not bound it: 2 MB of
64
+ * `<a><b><p>x</a>` is a million elements and seven seconds. This budget is about half a second at
65
+ * worst, and holds a 2 MB Wikipedia article (~25k) whole.
66
+ */
67
+ const MAX_ELEMENTS = 40_000
68
+ const DEPTH_PER_ELEMENT = 32
69
+
70
+ /** The most nodes converted in one piece; see {@link slices}. */
71
+ const SLICE_NODES = 2_000
72
+
73
+ /** Bytes the charset `<meta>` must sit within, as the HTML spec's prescan reads them. */
74
+ const CHARSET_PRESCAN_BYTES = 1024
75
+
76
+ type Parse5Node = DefaultTreeAdapterMap["node"]
77
+ type Parse5ParentNode = DefaultTreeAdapterMap["parentNode"]
78
+ type Parse5Document = DefaultTreeAdapterMap["document"]
79
+
80
+ /**
81
+ * Convert a fetched HTML page to markdown, keeping its content and dropping its chrome.
82
+ *
83
+ * `parse5` builds the tree the way a browser would, whatever the markup; the page is then pruned
84
+ * (scripts, navigation, hidden elements), narrowed to its single `main` or `article` when it has
85
+ * one, and written as GFM markdown. Links and images are absolute.
86
+ */
87
+ export function htmlToMarkdown(page: HtmlPage): HtmlMarkdown {
88
+ const html = decodeHtml(page.bytes, page.contentType)
89
+
90
+ const parsed = parseBounded(html)
91
+ const tree = fromParse5(parsed.document) as Root
92
+ const title = pageTitle(tree)
93
+ const base = baseUrl(tree, page.url)
94
+
95
+ const content = selectContent(prune(tree, base))
96
+ const markdown = collapseBlankLines(slices(content).map(writeMarkdown).join("\n\n"))
97
+
98
+ return { title, markdown, ...(parsed.truncated ? { truncated: true } : {}) }
99
+ }
100
+
101
+ function writeMarkdown(slice: Root): string {
102
+ const tree = toMdast(slice)
103
+ unwrapEmptyInline(tree)
104
+
105
+ return toMarkdown(tree, {
106
+ bullet: "-",
107
+ emphasis: "_",
108
+ fences: true,
109
+ listItemIndent: "one",
110
+ rule: "-",
111
+ extensions: [gfmToMarkdown()],
112
+ })
113
+ }
114
+
115
+ /** Inline nodes written as markers around their content. */
116
+ const WRAPPING_INLINE = new Set(["delete", "emphasis", "strong", "link"])
117
+
118
+ /**
119
+ * Replace inline nodes that show nothing by what they hold: written, an empty or blank one is its
120
+ * markers alone (`~~~~` for `<del></del>`), which at a line's start opens a code fence. Blank
121
+ * inline code keeps its spaces as text.
122
+ */
123
+ function unwrapEmptyInline(node: MdastNodes): void {
124
+ if (!("children" in node)) return
125
+
126
+ const children: MdastNodes[] = []
127
+ for (const child of node.children) {
128
+ unwrapEmptyInline(child)
129
+
130
+ if (child.type === "inlineCode" && !child.value.trim()) {
131
+ children.push({ type: "text", value: child.value })
132
+ } else if (WRAPPING_INLINE.has(child.type) && "children" in child && isBlank(child.children)) {
133
+ children.push(...child.children)
134
+ } else {
135
+ children.push(child)
136
+ }
137
+ }
138
+
139
+ node.children = children as typeof node.children
140
+ }
141
+
142
+ /** Only whitespace text, or nothing. */
143
+ function isBlank(nodes: readonly MdastNodes[]): boolean {
144
+ return nodes.every((node) => node.type === "text" && !node.value.trim())
145
+ }
146
+
147
+ /**
148
+ * Decode with the charset the header names, else the one a `<meta>` names near the top, else
149
+ * UTF-8. A byte-order mark wins over both, as in a browser. An unknown label decodes as UTF-8.
150
+ */
151
+ function decodeHtml(bytes: Uint8Array, contentType: string): string {
152
+ const label = bomCharset(bytes) ?? headerCharset(contentType) ?? metaCharset(bytes) ?? "utf-8"
153
+
154
+ return decoderFor(label).decode(bytes)
155
+ }
156
+
157
+ function bomCharset(bytes: Uint8Array): string | undefined {
158
+ if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) return "utf-8"
159
+ if (bytes[0] === 0xfe && bytes[1] === 0xff) return "utf-16be"
160
+ if (bytes[0] === 0xff && bytes[1] === 0xfe) return "utf-16le"
161
+
162
+ return undefined
163
+ }
164
+
165
+ function headerCharset(contentType: string): string | undefined {
166
+ return /charset\s*=\s*["']?([^"';\s]+)/i.exec(contentType)?.[1]
167
+ }
168
+
169
+ /** `<meta charset="…">` or `<meta http-equiv="content-type" content="…; charset=…">`. */
170
+ function metaCharset(bytes: Uint8Array): string | undefined {
171
+ // Latin-1 maps every byte to one character, so ASCII markup reads the same in any charset.
172
+ const head = new TextDecoder("latin1").decode(bytes.subarray(0, CHARSET_PRESCAN_BYTES))
173
+
174
+ return /<meta[^>]+charset\s*=\s*["']?([^"'>;\s/]+)/i.exec(head)?.[1]
175
+ }
176
+
177
+ function decoderFor(label: string): TextDecoder {
178
+ try {
179
+ return new TextDecoder(label.trim())
180
+ } catch {
181
+ return new TextDecoder("utf-8")
182
+ }
183
+ }
184
+
185
+ /**
186
+ * Parse like a browser, but stop the document where it nests deeper than {@link MAX_DEPTH} or
187
+ * places more than {@link MAX_ELEMENTS} elements.
188
+ *
189
+ * Throwing from the tree adapter is the only way to bound parse5: it keeps every open element on
190
+ * a stack and scans that stack for most start tags, so its time grows with depth × tags.
191
+ */
192
+ function parseBounded(html: string): { document: Parse5Document; truncated: boolean } {
193
+ const depths = new WeakMap<Parse5Node, number>()
194
+ let document: Parse5Document | undefined
195
+ let elements = 0
196
+
197
+ const place = (parent: Parse5ParentNode, child: Parse5Node): void => {
198
+ if (!defaultTreeAdapter.isElementNode(child)) return
199
+
200
+ const depth = (depths.get(parent) ?? 0) + 1
201
+ if (depth > MAX_DEPTH) throw new Cut()
202
+
203
+ elements += 1 + depth / DEPTH_PER_ELEMENT
204
+ if (elements > MAX_ELEMENTS) throw new Cut()
205
+
206
+ depths.set(child, depth)
207
+ }
208
+
209
+ const treeAdapter: TreeAdapter<DefaultTreeAdapterMap> = {
210
+ ...defaultTreeAdapter,
211
+
212
+ createDocument() {
213
+ document = defaultTreeAdapter.createDocument()
214
+ return document
215
+ },
216
+
217
+ appendChild(parent, child) {
218
+ place(parent, child)
219
+ defaultTreeAdapter.appendChild(parent, child)
220
+ },
221
+
222
+ insertBefore(parent, child, reference) {
223
+ place(parent, child)
224
+ defaultTreeAdapter.insertBefore(parent, child, reference)
225
+ },
226
+
227
+ // A template's content is a fragment of its own; it starts at the template's depth.
228
+ setTemplateContent(template, content) {
229
+ depths.set(content, depths.get(template) ?? 0)
230
+ defaultTreeAdapter.setTemplateContent(template, content)
231
+ },
232
+ }
233
+
234
+ try {
235
+ return { document: parse(html, { treeAdapter }), truncated: false }
236
+ } catch (error) {
237
+ if (error instanceof Cut && document) return { document, truncated: true }
238
+ throw error
239
+ }
240
+ }
241
+
242
+ class Cut extends Error {}
243
+
244
+ /** `og:title`, else `<title>`, both from the head. */
245
+ function pageTitle(tree: Root): string {
246
+ const head = findElement(tree, "head")
247
+ if (!head) return ""
248
+
249
+ for (const node of head.children) {
250
+ if (!isElement(node, "meta")) continue
251
+
252
+ const property = node.properties.property ?? node.properties.name
253
+ const content = node.properties.content
254
+ if (property === "og:title" && typeof content === "string" && content.trim()) {
255
+ return content.trim()
256
+ }
257
+ }
258
+
259
+ const title = findElement(head, "title")
260
+ if (!title) return ""
261
+
262
+ return title.children
263
+ .map((child) => (child.type === "text" ? child.value : ""))
264
+ .join("")
265
+ .replace(/\s+/g, " ")
266
+ .trim()
267
+ }
268
+
269
+ /** The page's URL, or its `<base href>` resolved against it. */
270
+ function baseUrl(tree: Root, pageUrl: string | URL): URL | undefined {
271
+ const page = parseUrl(String(pageUrl))
272
+ if (!page) return undefined
273
+
274
+ const base = findElement(tree, "base")
275
+ const href = base?.properties.href
276
+
277
+ return (typeof href === "string" && parseUrl(href, page)) || page
278
+ }
279
+
280
+ /**
281
+ * Drop what is not content, and make every link and image absolute.
282
+ *
283
+ * A link that goes nowhere a reader can follow (`javascript:`, malformed) keeps its text; an
284
+ * image that is not on the web (`data:`, malformed) is dropped, since its bytes would only bloat
285
+ * the page.
286
+ */
287
+ function prune<T extends Root | Element>(node: T, base: URL | undefined): T {
288
+ const children: ElementContent[] = []
289
+
290
+ for (const child of node.children) {
291
+ if (child.type === "comment" || child.type === "doctype") continue
292
+
293
+ if (child.type !== "element") {
294
+ children.push(child)
295
+ continue
296
+ }
297
+
298
+ if (isDropped(child)) continue
299
+
300
+ if (child.tagName === "a" && typeof child.properties.href === "string") {
301
+ const href = webUrl(child.properties.href, base, ["http:", "https:", "mailto:"])
302
+
303
+ if (!href) {
304
+ for (const grandchild of prune(child, base).children) children.push(grandchild)
305
+ continue
306
+ }
307
+
308
+ child.properties.href = href
309
+ }
310
+
311
+ if (child.tagName === "img") {
312
+ const src = webUrl(String(child.properties.src ?? ""), base, ["http:", "https:"])
313
+ if (!src) continue
314
+
315
+ child.properties.src = src
316
+ }
317
+
318
+ children.push(prune(child, base))
319
+ }
320
+
321
+ node.children = children as T["children"]
322
+ return node
323
+ }
324
+
325
+ function isDropped(element: Element): boolean {
326
+ if (DROPPED_ELEMENTS.has(element.tagName)) return true
327
+ if (element.properties.hidden) return true
328
+
329
+ return String(element.properties.ariaHidden).toLowerCase() === "true"
330
+ }
331
+
332
+ /** The page's single `main`, else its single `article`, else the whole page. */
333
+ function selectContent(tree: Root): Root {
334
+ for (const tagName of ["main", "article"]) {
335
+ const found = findElements(tree, tagName)
336
+ if (found.length === 1) return { type: "root", children: found }
337
+ }
338
+
339
+ return tree
340
+ }
341
+
342
+ /**
343
+ * Cut the content into trees of at most {@link SLICE_NODES} nodes, converted one at a time.
344
+ *
345
+ * The syntax-tree utilities find a node's index with `indexOf` on its parent's children, so a
346
+ * parent with n children costs n²: 160k table rows take 25 s in one piece. A small page is one
347
+ * slice and converts unchanged; on a large one, an element too big for a slice is repeated
348
+ * around each part of its children — a long table becomes several tables, a long list several
349
+ * lists.
350
+ */
351
+ function slices(content: Root): Root[] {
352
+ return split(content, new WeakMap())
353
+ }
354
+
355
+ function split<T extends Root | Element>(node: T, sizes: WeakMap<Nodes, number>): T[] {
356
+ if (sizeOf(node, sizes) <= SLICE_NODES) return [node]
357
+
358
+ const parts: T[] = []
359
+ let group: Array<T["children"][number]> = []
360
+ let groupSize = 0
361
+
362
+ const flush = (): void => {
363
+ if (group.length === 0) return
364
+
365
+ const part = { ...node, children: group }
366
+ sizes.set(part, groupSize + 1)
367
+ parts.push(part)
368
+
369
+ group = []
370
+ groupSize = 0
371
+ }
372
+
373
+ for (const child of node.children) {
374
+ const pieces = child.type === "element" ? split(child, sizes) : [child]
375
+
376
+ for (const piece of pieces) {
377
+ const size = sizeOf(piece, sizes)
378
+ if (groupSize + size > SLICE_NODES) flush()
379
+
380
+ group.push(piece)
381
+ groupSize += size
382
+ }
383
+ }
384
+
385
+ flush()
386
+ return parts
387
+ }
388
+
389
+ /** Nodes in the subtree, counted once and remembered. */
390
+ function sizeOf(node: Nodes, sizes: WeakMap<Nodes, number>): number {
391
+ const known = sizes.get(node)
392
+ if (known !== undefined) return known
393
+
394
+ let size = 1
395
+ if ("children" in node) {
396
+ for (const child of node.children) size += sizeOf(child, sizes)
397
+ }
398
+
399
+ sizes.set(node, size)
400
+ return size
401
+ }
402
+
403
+ /**
404
+ * Collapse runs of blank lines to one and trim, leaving fenced code exactly as written: its blank
405
+ * lines are part of the code.
406
+ */
407
+ function collapseBlankLines(markdown: string): string {
408
+ const lines: string[] = []
409
+ let fence: string | undefined
410
+ let blank = false
411
+
412
+ for (const line of markdown.split("\n")) {
413
+ const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1]
414
+
415
+ if (fence) {
416
+ if (marker?.startsWith(fence) && line.trim() === marker) fence = undefined
417
+ lines.push(line)
418
+ continue
419
+ }
420
+
421
+ if (marker) fence = marker
422
+
423
+ const isBlank = line.trim() === ""
424
+ if (isBlank && blank) continue
425
+
426
+ blank = isBlank
427
+ lines.push(isBlank ? "" : line)
428
+ }
429
+
430
+ return lines.join("\n").trim()
431
+ }
432
+
433
+ function webUrl(value: string, base: URL | undefined, protocols: string[]): string | undefined {
434
+ const url = parseUrl(value.trim(), base)
435
+ if (!url || !protocols.includes(url.protocol)) return undefined
436
+
437
+ return url.href
438
+ }
439
+
440
+ function parseUrl(value: string, base?: URL): URL | undefined {
441
+ try {
442
+ return new URL(value, base)
443
+ } catch {
444
+ return undefined
445
+ }
446
+ }
447
+
448
+ function isElement(node: RootContent | ElementContent, tagName: string): node is Element {
449
+ return node.type === "element" && node.tagName === tagName
450
+ }
451
+
452
+ function findElement(node: Root | Element, tagName: string): Element | undefined {
453
+ return findElements(node, tagName, 1)[0]
454
+ }
455
+
456
+ /** Elements named `tagName` under `node`, outermost first, without descending into a match. */
457
+ function findElements(node: Root | Element, tagName: string, limit = Infinity): Element[] {
458
+ const found: Element[] = []
459
+ const stack: Array<RootContent | ElementContent> = []
460
+ pushReversed(stack, node.children)
461
+
462
+ for (let child = stack.pop(); child && found.length < limit; child = stack.pop()) {
463
+ if (child.type !== "element") continue
464
+
465
+ if (child.tagName === tagName) {
466
+ found.push(child)
467
+ continue
468
+ }
469
+
470
+ pushReversed(stack, child.children)
471
+ }
472
+
473
+ return found
474
+ }
475
+
476
+ /** Push so that popping visits `items` in document order. No spread: a node may have millions. */
477
+ function pushReversed<T>(stack: T[], items: readonly T[]): void {
478
+ for (let index = items.length - 1; index >= 0; index--) stack.push(items[index] as T)
479
+ }