@cohortapp/agent-sdk 2.17.0 → 2.18.4

package/lib/engine/tools/webfetch-html.mjs

@@ -0,0 +1,387 @@
+/**
+ * lib/engine/tools/webfetch-html.mjs — HTML to readable Markdown (pure).
+ *
+ * An original, dependency-free converter aimed at what a model needs from a
+ * page: its text, headings, links, lists, code and tables — not a faithful
+ * rendering. It tokenises the markup into tags and text, drops non-content
+ * elements (script, style, head, svg, nav, footer, forms…), and emits
+ * Markdown with collapsed whitespace outside preformatted blocks. Relative
+ * links resolve against the page URL. Malformed markup degrades to text; it
+ * never throws.
+ *
+ * Linear time on hostile input. The conversion is synchronous, so a quadratic
+ * path would block the event loop (and with it every timeout and abort) for
+ * as long as a hostile page likes. Hence: no end-anchored regex over
+ * unbounded runs (trailing newlines and spaces are counted by hand), one
+ * sticky case-insensitive search per raw-text element instead of lowering the
+ * document, a link opened inside an open link closes the outer one (so
+ * closing never re-joins a growing tail), and list indentation is capped.
+ * `web-w3.test.mjs` times the adversarial shapes.
+ *
+ * @module lib/engine/tools/webfetch-html
+ */
+
+const SKIP = new Set(["script", "style", "noscript", "template", "head", "svg", "canvas", "iframe", "object", "embed", "nav", "footer", "form", "select", "button", "dialog"]);
+const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"]);
+const BLOCK = new Set(["p", "div", "section", "article", "main", "header", "aside", "figure", "figcaption", "address", "details", "summary", "dl", "dt", "dd", "table", "thead", "tbody", "tfoot", "center"]);
+
+/** Deeper lists are indented as this depth (indentation per item would otherwise grow with nesting). */
+const MAX_LIST_INDENT = 8;
+/** Raw-text elements, each with a sticky case-insensitive search for its closing tag. */
+const RAW_CLOSE = { script: /<\/script/iy, style: /<\/style/iy, textarea: /<\/textarea/iy, title: /<\/title/iy };
+
+/** Index of the next `</name` at or after `from` (case-insensitive), or -1. @param {string} html @param {RegExp} sticky @param {number} from */
+function findClose(html, sticky, from) {
+  const n = html.length;
+  let at = from;
+  while (at < n) {
+    const lt = html.indexOf("</", at);
+    if (lt === -1) return -1;
+    sticky.lastIndex = lt;
+    if (sticky.test(html)) return lt;
+    at = lt + 2;
+  }
+  return -1;
+}
+
+/** How many `\n` end `s`. @param {string} s */
+function trailingNewlines(s) {
+  let k = s.length;
+  while (k > 0 && s.charCodeAt(k - 1) === 10) k--;
+  return s.length - k;
+}
+
+/** Remove trailing spaces and tabs from every line, in one pass. @param {string} s */
+function trimLineEnds(s) {
+  let out = "";
+  let lineStart = 0;
+  const n = s.length;
+  for (let i = 0; i <= n; i++) {
+    if (i === n || s.charCodeAt(i) === 10) {
+      let end = i;
+      while (end > lineStart && (s.charCodeAt(end - 1) === 32 || s.charCodeAt(end - 1) === 9)) end--;
+      out += s.slice(lineStart, end);
+      if (i < n) out += "\n";
+      lineStart = i + 1;
+    }
+  }
+  return out;
+}
+
+const NAMED = {
+  amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", ndash: "–", mdash: "—", hellip: "…", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”",
+  copy: "©", reg: "®", trade: "™", middot: "·", bull: "•", laquo: "«", raquo: "»", times: "×", divide: "÷", deg: "°", euro: "€", pound: "£", yen: "¥", cent: "¢", sect: "§", para: "¶",
+};
+
+/** @param {string} s */
+export function decodeEntities(s) {
+  return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);?/gi, (m, e) => {
+    if (e[0] === "#") {
+      const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
+      return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : m;
+    }
+    const v = /** @type {Record<string,string>} */ (NAMED)[e.toLowerCase()];
+    return v ?? m;
+  });
+}
+
+/**
+ * @typedef {{type:'text', text:string} | {type:'open', name:string, attrs:Record<string,string>, selfClosing:boolean} | {type:'close', name:string}} Token
+ */
+
+/** @param {string} html @returns {Token[]} */
+export function tokenize(html) {
+  /** @type {Token[]} */
+  const out = [];
+  let i = 0;
+  const n = html.length;
+  while (i < n) {
+    const lt = html.indexOf("<", i);
+    if (lt === -1) {
+      out.push({ type: "text", text: html.slice(i) });
+      break;
+    }
+    if (lt > i) out.push({ type: "text", text: html.slice(i, lt) });
+    if (html.startsWith("<!--", lt)) {
+      const end = html.indexOf("-->", lt + 4);
+      i = end === -1 ? n : end + 3;
+      continue;
+    }
+    if (html[lt + 1] === "!" || html[lt + 1] === "?") {
+      const end = html.indexOf(">", lt);
+      i = end === -1 ? n : end + 1;
+      continue;
+    }
+    const m = /^<(\/?)([a-zA-Z][a-zA-Z0-9:-]*)/.exec(html.slice(lt, lt + 64));
+    if (!m) {
+      out.push({ type: "text", text: "<" });
+      i = lt + 1;
+      continue;
+    }
+    // Find the end of the tag, honouring quoted attribute values.
+    let j = lt + m[0].length;
+    let quote = null;
+    while (j < n) {
+      const ch = html[j];
+      if (quote) {
+        if (ch === quote) quote = null;
+      } else if (ch === '"' || ch === "'") quote = ch;
+      else if (ch === ">") break;
+      j++;
+    }
+    const inner = html.slice(lt + m[0].length, j);
+    const name = m[2].toLowerCase();
+    i = j + 1;
+    if (m[1] === "/") {
+      out.push({ type: "close", name });
+      continue;
+    }
+    /** @type {Record<string,string>} */
+    const attrs = {};
+    const re = /([^\s=/>]+)(?:\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+)))?/g;
+    let a;
+    while ((a = re.exec(inner)) !== null) attrs[a[1].toLowerCase()] = decodeEntities(a[3] ?? a[4] ?? a[5] ?? "");
+    out.push({ type: "open", name, attrs, selfClosing: /\/\s*$/.test(inner) || VOID.has(name) });
+    // Raw-text elements: their content is not markup.
+    const rawClose = /** @type {Record<string, RegExp>} */ (RAW_CLOSE)[name];
+    if (rawClose && !/\/\s*$/.test(inner)) {
+      const close = findClose(html, rawClose, i);
+      const end = close === -1 ? n : close;
+      out.push({ type: "text", text: html.slice(i, end) });
+      i = end;
+    }
+  }
+  return out;
+}
+
+/** @param {string} href @param {string|undefined} base */
+function resolveHref(href, base) {
+  if (!href) return "";
+  if (/^(javascript|data|vbscript):/i.test(href.trim())) return "";
+  try {
+    return base ? new URL(href, base).href : href;
+  } catch {
+    return href;
+  }
+}
+
+/**
+ * @param {string} html
+ * @param {{baseUrl?:string}} [o]
+ * @returns {{title:string|null, markdown:string}}
+ */
+export function htmlToMarkdown(html, { baseUrl } = {}) {
+  const tokens = tokenize(String(html ?? ""));
+  let title = null;
+  /** @type {string[]} */
+  const out = [];
+  const skipStack = [];
+  let pre = 0;
+  const lists = [];
+  /** @type {Array<{href:string, start:number}>} */
+  const links = [];
+  let table = null;
+
+  // The tail of the output, tracked incrementally (joining on every newline
+  // would be quadratic on a large page).
+  let lastChar = "";
+  let trailingNl = 0;
+  const track = (/** @type string */ s) => {
+    const nl = trailingNewlines(s);
+    trailingNl = nl === s.length ? trailingNl + nl : nl;
+    lastChar = s[s.length - 1];
+  };
+  // After a truncation: scan back only as far as the tail needs — newline()
+  // never asks for more than two, so two trailing newlines end the scan (a
+  // long run of newline-only entries would otherwise be rescanned per link).
+  const retrack = () => {
+    lastChar = "";
+    trailingNl = 0;
+    for (let j = out.length - 1; j >= 0; j--) {
+      const s = out[j];
+      if (s === "") continue;
+      if (lastChar === "") lastChar = s[s.length - 1];
+      const nl = trailingNewlines(s);
+      trailingNl += nl;
+      if (nl !== s.length || trailingNl >= 2) break;
+    }
+  };
+  const closeLink = () => {
+    const link = links.pop();
+    if (!link) return;
+    // In place: copying the whole output per link is quadratic on link-heavy pages.
+    if (link.start >= out.length) return;
+    const text = out.splice(link.start).join("").trim();
+    retrack();
+    if (link.href && text) emit(`[${text}](${link.href})`);
+    else emit(text);
+  };
+  const emit = (/** @type string */ s) => {
+    if (s === "") return;
+    out.push(s);
+    track(s);
+  };
+  const newline = (/** @type number */ count = 1) => {
+    if (lastChar === "") return;
+    if (trailingNl < count) emit("\n".repeat(count - trailingNl));
+  };
+
+  for (let k = 0; k < tokens.length; k++) {
+    const t = tokens[k];
+    if (t.type === "open" && t.name === "title" && title === null) {
+      const next = tokens[k + 1];
+      if (next?.type === "text") title = decodeEntities(next.text).replace(/\s+/g, " ").trim() || null;
+    }
+    if (skipStack.length > 0) {
+      if (t.type === "open" && SKIP.has(t.name) && !t.selfClosing) skipStack.push(t.name);
+      else if (t.type === "close" && t.name === skipStack[skipStack.length - 1]) skipStack.pop();
+      continue;
+    }
+    if (t.type === "text") {
+      let text = decodeEntities(t.text);
+      if (pre > 0) {
+        emit(text);
+        continue;
+      }
+      text = text.replace(/[\s ]+/g, " ");
+      if (lastChar === "" || lastChar === "\n" || lastChar === " ") text = text.replace(/^ /, "");
+      if (text !== "") emit(table ? text.replace(/\|/g, "\\|") : text);
+      continue;
+    }
+    if (t.type === "open") {
+      if (SKIP.has(t.name)) {
+        if (!t.selfClosing) skipStack.push(t.name);
+        continue;
+      }
+      switch (t.name) {
+        case "h1": case "h2": case "h3": case "h4": case "h5": case "h6":
+          newline(2);
+          emit(`${"#".repeat(Number(t.name[1]))} `);
+          break;
+        case "br":
+          emit(pre > 0 ? "\n" : "  \n");
+          break;
+        case "hr":
+          newline(2);
+          emit("---");
+          newline(2);
+          break;
+        case "pre":
+          newline(2);
+          emit("```\n");
+          pre++;
+          break;
+        case "code":
+          if (pre === 0) emit("`");
+          break;
+        case "strong": case "b":
+          emit("**");
+          break;
+        case "em": case "i":
+          emit("*");
+          break;
+        case "blockquote":
+          newline(2);
+          emit("> ");
+          break;
+        case "ul": case "ol":
+          newline(1);
+          lists.push({ ordered: t.name === "ol", n: 0 });
+          break;
+        case "li": {
+          newline(1);
+          const l = lists[lists.length - 1];
+          const indent = "  ".repeat(Math.min(MAX_LIST_INDENT, Math.max(0, lists.length - 1)));
+          if (l) l.n++;
+          emit(`${indent}${l?.ordered ? `${l.n}.` : "-"} `);
+          break;
+        }
+        case "a":
+          // Links do not nest in HTML: a new one ends the open one.
+          closeLink();
+          links.push({ href: resolveHref(t.attrs.href ?? "", baseUrl), start: out.length });
+          break;
+        case "img": {
+          const alt = (t.attrs.alt ?? "").trim();
+          if (alt) emit(`[image: ${alt}]`);
+          break;
+        }
+        case "tr":
+          if (table) {
+            newline(1);
+            emit("|");
+            table.cells = 0;
+          }
+          break;
+        case "th": case "td":
+          if (table) emit(" ");
+          break;
+        default:
+          if (t.name === "table") {
+            newline(2);
+            table = { rows: 0, cells: 0, headerDone: false };
+          } else if (BLOCK.has(t.name)) newline(t.name === "p" ? 2 : 1);
+      }
+      continue;
+    }
+    // close
+    switch (t.name) {
+      case "h1": case "h2": case "h3": case "h4": case "h5": case "h6": case "p":
+        newline(2);
+        break;
+      case "pre":
+        if (pre > 0) {
+          pre--;
+          newline(1);
+          emit("```");
+          newline(2);
+        }
+        break;
+      case "code":
+        if (pre === 0) emit("`");
+        break;
+      case "strong": case "b":
+        emit("**");
+        break;
+      case "em": case "i":
+        emit("*");
+        break;
+      case "ul": case "ol":
+        lists.pop();
+        newline(lists.length === 0 ? 2 : 1);
+        break;
+      case "blockquote":
+        newline(2);
+        break;
+      case "a":
+        closeLink();
+        break;
+      case "th": case "td":
+        if (table) {
+          emit(" |");
+          table.cells++;
+        }
+        break;
+      case "tr":
+        if (table && !table.headerDone && table.cells > 0) {
+          newline(1);
+          emit(`|${" --- |".repeat(table.cells)}`);
+          table.headerDone = true;
+        }
+        break;
+      case "table":
+        table = null;
+        newline(2);
+        break;
+      default:
+        if (BLOCK.has(t.name)) newline(1);
+    }
+  }
+
+  closeLink(); // a link left open at the end of the page
+  const markdown = trimLineEnds(out.join(""))
+    .replace(/\n{3,}/g, "\n\n")
+    .replace(/\*\*\s*\*\*/g, "")
+    .trim();
+  return { title, markdown };
+}