@sembl/source-html 0.3.0 → 0.5.0

package/README.md

@@ -18,6 +18,15 @@ const listing = await coerce<Listing>(htmlSource(html, "Airbnb listing"), {
 });
 ```
 
+`htmlSources(html, label)` returns the same content as two labelled sources
+— the structured data and the readable text — so that SEMBL's budget, which
+cuts long sources first, can never truncate the JSON-LD away however large
+the page. `extractImages(html, { baseUrl })` returns the page's images for a
+gallery: OpenGraph and Twitter card images first, then JSON-LD `image`
+values, then `<img>` tags including lazy-loaded ones and the widest `srcset`
+candidate, with tracking pixels, icons, logos, sprites and `data:` URIs
+dropped and duplicates folded.
+
 `htmlSource` renders the page in three sections, in this order:
 
 1. **Page metadata** — the `<title>` and every `<meta>` tag keyed by

package/dist/index.cjs

@@ -21,9 +21,11 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var index_exports = {};
 __export(index_exports, {
   decodeEntities: () => decodeEntities,
+  extractImages: () => extractImages,
   extractJsonLd: () => extractJsonLd,
   extractMeta: () => extractMeta,
   htmlSource: () => htmlSource,
+  htmlSources: () => htmlSources,
   htmlToText: () => htmlToText,
   pageToText: () => pageToText,
   preprocessHtml: () => preprocessHtml
@@ -193,6 +195,92 @@ ${text}`);
   }
   return sections.join("\n\n");
 }
+function htmlSources(html, label = "Page", options = {}) {
+  const { body = true } = options;
+  const structured = pageToText(html, { ...options, body: false });
+  const text = body ? pageToText(html, { meta: false, jsonLd: false, body: true }) : "";
+  const sources = [];
+  if (structured) sources.push({ label: `${label} (structured data)`, text: structured });
+  if (text) sources.push({ label, text });
+  return sources;
+}
+var JUNK = /(?:^|[\/_.-])(?:logo|icon|favicon|sprite|avatar|badge|pixel|tracking|blank|spacer|placeholder|loading|spinner|arrow|flag|emoji|button|banner-ad|ads?)(?:[\/_.-]|$)/i;
+function toAbsolute(url, baseUrl) {
+  const trimmed = decodeEntities(url.trim());
+  if (!trimmed || trimmed.startsWith("data:") || trimmed.startsWith("blob:") || trimmed.startsWith("javascript:")) return void 0;
+  try {
+    const absolute = baseUrl ? new URL(trimmed, baseUrl) : new URL(trimmed);
+    if (absolute.protocol !== "http:" && absolute.protocol !== "https:") return void 0;
+    return absolute.href;
+  } catch {
+    return void 0;
+  }
+}
+function isJunk(url) {
+  const path = url.replace(/^https?:\/\/[^/]+/, "").split("?")[0];
+  return JUNK.test(path) || /\.(?:svg|gif|ico|bmp)$/i.test(path);
+}
+function attr(attrs, name) {
+  const match = new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'>]+))`, "i").exec(attrs);
+  return match ? match[1] ?? match[2] ?? match[3] : void 0;
+}
+function dimension(value) {
+  if (!value) return void 0;
+  const n = parseInt(value, 10);
+  return Number.isFinite(n) ? n : void 0;
+}
+function extractImages(html, options = {}) {
+  const { baseUrl, max = 50, minSize = 100 } = options;
+  const seen = /* @__PURE__ */ new Set();
+  const images = [];
+  const add = (raw, extra = {}) => {
+    if (!raw || images.length >= max) return;
+    const url = toAbsolute(raw, baseUrl);
+    if (!url || seen.has(url) || isJunk(url)) return;
+    if (extra.width !== void 0 && extra.width < minSize || extra.height !== void 0 && extra.height < minSize) return;
+    seen.add(url);
+    const image = { url };
+    if (extra.alt) image.alt = extra.alt;
+    if (extra.width !== void 0) image.width = extra.width;
+    if (extra.height !== void 0) image.height = extra.height;
+    images.push(image);
+  };
+  const meta = extractMeta(html);
+  for (const key of ["og:image", "og:image:secure_url", "twitter:image", "twitter:image:src"]) {
+    add(meta[key]);
+  }
+  const visit = (value) => {
+    if (typeof value === "string") add(value);
+    else if (Array.isArray(value)) value.forEach(visit);
+    else if (value && typeof value === "object") {
+      const record = value;
+      if (typeof record.url === "string" && (record["@type"] === "ImageObject" || "contentUrl" in record)) add(record.url);
+      if (typeof record.contentUrl === "string") add(record.contentUrl);
+      if ("image" in record) visit(record.image);
+      if ("photo" in record) visit(record.photo);
+      for (const [key, child] of Object.entries(record)) {
+        if (key !== "image" && key !== "photo" && child && typeof child === "object") visit(child);
+      }
+    }
+  };
+  for (const block of extractJsonLd(html)) visit(block);
+  const tags = /<img\b([^>]*)>/gi;
+  let match;
+  while ((match = tags.exec(html)) !== null) {
+    const attrs = match[1];
+    const width = dimension(attr(attrs, "width"));
+    const height = dimension(attr(attrs, "height"));
+    const alt = attr(attrs, "alt")?.trim();
+    const srcset = attr(attrs, "srcset") ?? attr(attrs, "data-srcset");
+    let candidate = attr(attrs, "src") ?? attr(attrs, "data-src") ?? attr(attrs, "data-lazy-src");
+    if (srcset) {
+      const best = srcset.split(",").map((entry) => entry.trim().split(/\s+/)).map(([url, size]) => ({ url, w: size?.endsWith("w") ? parseInt(size, 10) : 0 })).sort((a, b) => b.w - a.w)[0];
+      if (best?.url) candidate = best.url;
+    }
+    add(candidate, { alt: alt || void 0, width, height });
+  }
+  return images;
+}
 function htmlSource(html, label, options) {
   const text = pageToText(html, options);
   return label ? { label, text } : { text };
@@ -203,9 +291,11 @@ function preprocessHtml(options) {
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   decodeEntities,
+  extractImages,
   extractJsonLd,
   extractMeta,
   htmlSource,
+  htmlSources,
   htmlToText,
   pageToText,
   preprocessHtml

package/dist/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts","../src/html-to-text.ts"],"sourcesContent":["import type { Source } from \"@sembl/core\";\nimport { extractJsonLd, extractMeta, htmlToText } from \"./html-to-text.js\";\n\nexport { htmlToText, extractJsonLd, extractMeta, decodeEntities } from \"./html-to-text.js\";\n\n/** Options for {@link htmlSource} and {@link pageToText}. */\nexport interface HtmlSourceOptions {\n  /** Include JSON-LD blocks ahead of the body text. Default true. */\n  jsonLd?: boolean;\n  /** Include the title and meta tags ahead of the body text. Default true. */\n  meta?: boolean;\n  /** Include the body text. Default true. */\n  body?: boolean;\n}\n\n/**\n * Render a page as text for extraction.\n *\n * Structured data comes first — the title and meta tags, then any JSON-LD —\n * and the readable body last. That order is deliberate: SEMBL's default\n * truncation keeps the head of a source, so on a page that blows the input\n * budget the parts most likely to hold clean facts are the parts that\n * survive.\n */\nexport function pageToText(html: string, options: HtmlSourceOptions = {}): string {\n  const { jsonLd = true, meta = true, body = true } = options;\n  const sections: string[] = [];\n\n  if (meta) {\n    const tags = extractMeta(html);\n    const lines = Object.entries(tags).map(([key, value]) => `${key}: ${value}`);\n    if (lines.length > 0) sections.push(`Page metadata:\\n${lines.join(\"\\n\")}`);\n  }\n\n  if (jsonLd) {\n    const blocks = extractJsonLd(html);\n    if (blocks.length > 0) {\n      sections.push(\n        `Structured data (JSON-LD):\\n${blocks.map((b) => JSON.stringify(b)).join(\"\\n\")}`,\n      );\n    }\n  }\n\n  if (body) {\n    const text = htmlToText(html);\n    if (text) sections.push(`Page text:\\n${text}`);\n  }\n\n  return sections.join(\"\\n\\n\");\n}\n\n/**\n * Build a labelled SEMBL source from a page, ready to pass to any coercion or\n * to `sembl()`.\n */\nexport function htmlSource(html: string, label?: string, options?: HtmlSourceOptions): Source {\n  const text = pageToText(html, options);\n  return label ? { label, text } : { text };\n}\n\n/**\n * A `preprocess` hook that converts every source's text from HTML, for the\n * case where the sources are pages but you would rather keep the fetch and\n * the coercion apart:\n *\n * ```ts\n * await coerce(pages, { provider, schema, preprocess: preprocessHtml() });\n * ```\n */\nexport function preprocessHtml(options?: HtmlSourceOptions): (source: Source) => Source {\n  return (source) => ({ ...source, text: pageToText(source.text, options) });\n}\n","/**\n * A small, dependency-free HTML-to-text pass tuned for feeding a page to a\n * language model rather than for rendering it.\n *\n * It is regex-based, so it is not a parser: malformed markup degrades to\n * slightly worse text rather than to an error, which is the right trade for\n * scraped input. Structured data the page already carries — JSON-LD blocks,\n * OpenGraph and meta tags, the title — is pulled out separately so it can be\n * placed ahead of the body text, where head-keeping truncation preserves it.\n */\n\n/** Elements whose contents never carry readable text. */\nconst DROP_ELEMENTS = [\"script\", \"style\", \"noscript\", \"template\", \"svg\", \"iframe\", \"head\"];\n\n/** Elements that end a line when they open or close. */\nconst BLOCK_ELEMENTS = [\n  \"address\", \"article\", \"aside\", \"blockquote\", \"dd\", \"details\", \"dialog\", \"div\", \"dl\", \"dt\",\n  \"fieldset\", \"figcaption\", \"figure\", \"footer\", \"form\", \"h1\", \"h2\", \"h3\", \"h4\", \"h5\", \"h6\",\n  \"header\", \"hr\", \"main\", \"nav\", \"ol\", \"p\", \"pre\", \"section\", \"summary\", \"table\", \"tbody\",\n  \"td\", \"tfoot\", \"th\", \"thead\", \"tr\", \"ul\",\n];\n\nconst NAMED_ENTITIES: Record<string, string> = {\n  amp: \"&\", lt: \"<\", gt: \">\", quot: '\"', apos: \"'\", nbsp: \" \", copy: \"©\", reg: \"®\",\n  trade: \"™\", hellip: \"…\", mdash: \"—\", ndash: \"–\", lsquo: \"‘\", rsquo: \"’\", ldquo: \"“\",\n  rdquo: \"”\", bull: \"•\", middot: \"·\", deg: \"°\", euro: \"€\", pound: \"£\", yen: \"¥\", cent: \"¢\",\n  frac12: \"½\", frac14: \"¼\", frac34: \"¾\", times: \"×\", laquo: \"«\", raquo: \"»\",\n};\n\n/** Decode numeric and the common named character references. */\nexport function decodeEntities(text: string): string {\n  return text.replace(/&(#x[0-9a-f]+|#\\d+|[a-z][a-z0-9]*);/gi, (match, ref: string) => {\n    if (ref[0] === \"#\") {\n      const code = ref[1].toLowerCase() === \"x\" ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10);\n      return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match;\n    }\n    return NAMED_ENTITIES[ref.toLowerCase()] ?? match;\n  });\n}\n\n/** Remove an element and everything inside it, for each name given. */\nfunction dropElements(html: string, names: readonly string[]): string {\n  return names.reduce(\n    (acc, name) => acc.replace(new RegExp(`<${name}\\\\b[^>]*>[\\\\s\\\\S]*?</${name}\\\\s*>`, \"gi\"), \" \"),\n    html,\n  );\n}\n\n/**\n * Reduce a page's markup to readable text: comments and non-text elements\n * removed, block boundaries turned into line breaks, list items bulleted,\n * entities decoded, whitespace collapsed.\n */\nexport function htmlToText(html: string): string {\n  let text = html.replace(/<!--[\\s\\S]*?-->/g, \" \");\n  text = dropElements(text, DROP_ELEMENTS);\n  text = text.replace(/<br\\s*\\/?>/gi, \"\\n\");\n  text = text.replace(/<li\\b[^>]*>/gi, \"\\n- \");\n  text = text.replace(new RegExp(`</?(?:${BLOCK_ELEMENTS.join(\"|\")})\\\\b[^>]*>`, \"gi\"), \"\\n\");\n  text = text.replace(/<[^>]+>/g, \" \");\n  text = decodeEntities(text);\n  text = text.replace(/[ \\t\\f\\v ]+/g, \" \");\n  text = text\n    .split(\"\\n\")\n    .map((line) => line.trim())\n    .join(\"\\n\")\n    .replace(/\\n{3,}/g, \"\\n\\n\");\n  return text.trim();\n}\n\n/**\n * Every parseable `<script type=\"application/ld+json\">` block on the page.\n * A block that fails to parse is skipped: it is the page's bug, not ours.\n */\nexport function extractJsonLd(html: string): unknown[] {\n  const blocks: unknown[] = [];\n  const pattern = /<script\\b[^>]*type\\s*=\\s*[\"']?application\\/ld\\+json[\"']?[^>]*>([\\s\\S]*?)<\\/script\\s*>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    try {\n      blocks.push(JSON.parse(match[1].trim()));\n    } catch {\n      // Skip a malformed block rather than lose the rest of the page.\n    }\n  }\n  return blocks;\n}\n\n/** Read one attribute off a tag's attribute string. */\nfunction attribute(attrs: string, name: string): string | undefined {\n  const match = new RegExp(`\\\\b${name}\\\\s*=\\\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\\\s\"'>]+))`, \"i\").exec(attrs);\n  if (!match) return undefined;\n  return decodeEntities(match[1] ?? match[2] ?? match[3] ?? \"\");\n}\n\n/**\n * The page's `<title>` and its `<meta>` tags keyed by `property` or `name`\n * — OpenGraph (`og:*`), Twitter cards, `description`, and so on. Later tags\n * with the same key win, matching how most scrapers read them.\n */\nexport function extractMeta(html: string): Record<string, string> {\n  const meta: Record<string, string> = {};\n  const title = /<title\\b[^>]*>([\\s\\S]*?)<\\/title\\s*>/i.exec(html);\n  if (title) {\n    const text = decodeEntities(title[1]).replace(/\\s+/g, \" \").trim();\n    if (text) meta.title = text;\n  }\n  const pattern = /<meta\\b([^>]*)>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    const attrs = match[1];\n    const key = attribute(attrs, \"property\") ?? attribute(attrs, \"name\");\n    const content = attribute(attrs, \"content\");\n    if (key && content !== undefined && content.trim()) {\n      meta[key.toLowerCase()] = content.trim();\n    }\n  }\n  return meta;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACYA,IAAM,gBAAgB,CAAC,UAAU,SAAS,YAAY,YAAY,OAAO,UAAU,MAAM;AAGzF,IAAM,iBAAiB;AAAA,EACrB;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAAc;AAAA,EAAM;AAAA,EAAW;AAAA,EAAU;AAAA,EAAO;AAAA,EAAM;AAAA,EACrF;AAAA,EAAY;AAAA,EAAc;AAAA,EAAU;AAAA,EAAU;AAAA,EAAQ;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EACpF;AAAA,EAAU;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAM;AAAA,EAAK;AAAA,EAAO;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAChF;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AACtC;AAEA,IAAM,iBAAyC;AAAA,EAC7C,KAAK;AAAA,EAAK,IAAI;AAAA,EAAK,IAAI;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,KAAK;AAAA,EAC7E,OAAO;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAChF,OAAO;AAAA,EAAK,MAAM;AAAA,EAAK,QAAQ;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EAAK,OAAO;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EACrF,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AACxE;AAGO,SAAS,eAAe,MAAsB;AACnD,SAAO,KAAK,QAAQ,yCAAyC,CAAC,OAAO,QAAgB;AACnF,QAAI,IAAI,CAAC,MAAM,KAAK;AAClB,YAAM,OAAO,IAAI,CAAC,EAAE,YAAY,MAAM,MAAM,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE;AAClG,aAAO,OAAO,SAAS,IAAI,KAAK,OAAO,KAAK,QAAQ,UAAW,OAAO,cAAc,IAAI,IAAI;AAAA,IAC9F;AACA,WAAO,eAAe,IAAI,YAAY,CAAC,KAAK;AAAA,EAC9C,CAAC;AACH;AAGA,SAAS,aAAa,MAAc,OAAkC;AACpE,SAAO,MAAM;AAAA,IACX,CAAC,KAAK,SAAS,IAAI,QAAQ,IAAI,OAAO,IAAI,IAAI,wBAAwB,IAAI,SAAS,IAAI,GAAG,GAAG;AAAA,IAC7F;AAAA,EACF;AACF;AAOO,SAAS,WAAW,MAAsB;AAC/C,MAAI,OAAO,KAAK,QAAQ,oBAAoB,GAAG;AAC/C,SAAO,aAAa,MAAM,aAAa;AACvC,SAAO,KAAK,QAAQ,gBAAgB,IAAI;AACxC,SAAO,KAAK,QAAQ,iBAAiB,MAAM;AAC3C,SAAO,KAAK,QAAQ,IAAI,OAAO,SAAS,eAAe,KAAK,GAAG,CAAC,cAAc,IAAI,GAAG,IAAI;AACzF,SAAO,KAAK,QAAQ,YAAY,GAAG;AACnC,SAAO,eAAe,IAAI;AAC1B,SAAO,KAAK,QAAQ,gBAAgB,GAAG;AACvC,SAAO,KACJ,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,KAAK,IAAI,EACT,QAAQ,WAAW,MAAM;AAC5B,SAAO,KAAK,KAAK;AACnB;AAMO,SAAS,cAAc,MAAyB;AACrD,QAAM,SAAoB,CAAC;AAC3B,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,QAAI;AACF,aAAO,KAAK,KAAK,MAAM,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC;AAAA,IACzC,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAGA,SAAS,UAAU,OAAe,MAAkC;AAClE,QAAM,QAAQ,IAAI,OAAO,MAAM,IAAI,iDAAiD,GAAG,EAAE,KAAK,KAAK;AACnG,MAAI,CAAC,MAAO,QAAO;AACnB,SAAO,eAAe,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,EAAE;AAC9D;AAOO,SAAS,YAAY,MAAsC;AAChE,QAAM,OAA+B,CAAC;AACtC,QAAM,QAAQ,wCAAwC,KAAK,IAAI;AAC/D,MAAI,OAAO;AACT,UAAM,OAAO,eAAe,MAAM,CAAC,CAAC,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAChE,QAAI,KAAM,MAAK,QAAQ;AAAA,EACzB;AACA,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,UAAM,QAAQ,MAAM,CAAC;AACrB,UAAM,MAAM,UAAU,OAAO,UAAU,KAAK,UAAU,OAAO,MAAM;AACnE,UAAM,UAAU,UAAU,OAAO,SAAS;AAC1C,QAAI,OAAO,YAAY,UAAa,QAAQ,KAAK,GAAG;AAClD,WAAK,IAAI,YAAY,CAAC,IAAI,QAAQ,KAAK;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;;;AD9FO,SAAS,WAAW,MAAc,UAA6B,CAAC,GAAW;AAChF,QAAM,EAAE,SAAS,MAAM,OAAO,MAAM,OAAO,KAAK,IAAI;AACpD,QAAM,WAAqB,CAAC;AAE5B,MAAI,MAAM;AACR,UAAM,OAAO,YAAY,IAAI;AAC7B,UAAM,QAAQ,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE;AAC3E,QAAI,MAAM,SAAS,EAAG,UAAS,KAAK;AAAA,EAAmB,MAAM,KAAK,IAAI,CAAC,EAAE;AAAA,EAC3E;AAEA,MAAI,QAAQ;AACV,UAAM,SAAS,cAAc,IAAI;AACjC,QAAI,OAAO,SAAS,GAAG;AACrB,eAAS;AAAA,QACP;AAAA,EAA+B,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,MAChF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,MAAM;AACR,UAAM,OAAO,WAAW,IAAI;AAC5B,QAAI,KAAM,UAAS,KAAK;AAAA,EAAe,IAAI,EAAE;AAAA,EAC/C;AAEA,SAAO,SAAS,KAAK,MAAM;AAC7B;AAMO,SAAS,WAAW,MAAc,OAAgB,SAAqC;AAC5F,QAAM,OAAO,WAAW,MAAM,OAAO;AACrC,SAAO,QAAQ,EAAE,OAAO,KAAK,IAAI,EAAE,KAAK;AAC1C;AAWO,SAAS,eAAe,SAAyD;AACtF,SAAO,CAAC,YAAY,EAAE,GAAG,QAAQ,MAAM,WAAW,OAAO,MAAM,OAAO,EAAE;AAC1E;","names":[]}
+{"version":3,"sources":["../src/index.ts","../src/html-to-text.ts"],"sourcesContent":["import type { Source } from \"@sembl/core\";\nimport { decodeEntities, extractJsonLd, extractMeta, htmlToText } from \"./html-to-text.js\";\n\nexport { htmlToText, extractJsonLd, extractMeta, decodeEntities } from \"./html-to-text.js\";\n\n/** Options for {@link htmlSource} and {@link pageToText}. */\nexport interface HtmlSourceOptions {\n  /** Include JSON-LD blocks ahead of the body text. Default true. */\n  jsonLd?: boolean;\n  /** Include the title and meta tags ahead of the body text. Default true. */\n  meta?: boolean;\n  /** Include the body text. Default true. */\n  body?: boolean;\n}\n\n/**\n * Render a page as text for extraction.\n *\n * Structured data comes first — the title and meta tags, then any JSON-LD —\n * and the readable body last. That order is deliberate: SEMBL's default\n * truncation keeps the head of a source, so on a page that blows the input\n * budget the parts most likely to hold clean facts are the parts that\n * survive.\n */\nexport function pageToText(html: string, options: HtmlSourceOptions = {}): string {\n  const { jsonLd = true, meta = true, body = true } = options;\n  const sections: string[] = [];\n\n  if (meta) {\n    const tags = extractMeta(html);\n    const lines = Object.entries(tags).map(([key, value]) => `${key}: ${value}`);\n    if (lines.length > 0) sections.push(`Page metadata:\\n${lines.join(\"\\n\")}`);\n  }\n\n  if (jsonLd) {\n    const blocks = extractJsonLd(html);\n    if (blocks.length > 0) {\n      sections.push(\n        `Structured data (JSON-LD):\\n${blocks.map((b) => JSON.stringify(b)).join(\"\\n\")}`,\n      );\n    }\n  }\n\n  if (body) {\n    const text = htmlToText(html);\n    if (text) sections.push(`Page text:\\n${text}`);\n  }\n\n  return sections.join(\"\\n\\n\");\n}\n\n/**\n * A page as two sources: the structured data (title, meta tags, JSON-LD)\n * and the readable text, each labelled. The structured source is short, so\n * SEMBL's budget — which cuts long sources first — never touches it; a huge\n * page loses body text, never its JSON-LD. Pass the result straight to a\n * coercion. Only the sources that have content are returned.\n */\nexport function htmlSources(html: string, label = \"Page\", options: HtmlSourceOptions = {}): Source[] {\n  const { body = true } = options;\n  const structured = pageToText(html, { ...options, body: false });\n  const text = body ? pageToText(html, { meta: false, jsonLd: false, body: true }) : \"\";\n  const sources: Source[] = [];\n  if (structured) sources.push({ label: `${label} (structured data)`, text: structured });\n  if (text) sources.push({ label, text });\n  return sources;\n}\n\n/** An image the page shows, with whatever size hints it gave. */\nexport interface HarvestedImage {\n  url: string;\n  alt?: string;\n  width?: number;\n  height?: number;\n}\n\n/** Options for {@link extractImages}. */\nexport interface ExtractImagesOptions {\n  /** Resolves relative URLs. Without it, relative URLs are dropped. */\n  baseUrl?: string;\n  /** Most images to return, in page order with meta images first. Default 50. */\n  max?: number;\n  /**\n   * Drop images whose declared width or height is under this many pixels.\n   * Undeclared sizes are kept. Default 100.\n   */\n  minSize?: number;\n}\n\n/** Path words that mark chrome rather than content. */\nconst JUNK = /(?:^|[\\/_.-])(?:logo|icon|favicon|sprite|avatar|badge|pixel|tracking|blank|spacer|placeholder|loading|spinner|arrow|flag|emoji|button|banner-ad|ads?)(?:[\\/_.-]|$)/i;\n\nfunction toAbsolute(url: string, baseUrl: string | undefined): string | undefined {\n  const trimmed = decodeEntities(url.trim());\n  if (!trimmed || trimmed.startsWith(\"data:\") || trimmed.startsWith(\"blob:\") || trimmed.startsWith(\"javascript:\")) return undefined;\n  try {\n    const absolute = baseUrl ? new URL(trimmed, baseUrl) : new URL(trimmed);\n    if (absolute.protocol !== \"http:\" && absolute.protocol !== \"https:\") return undefined;\n    return absolute.href;\n  } catch {\n    return undefined;\n  }\n}\n\nfunction isJunk(url: string): boolean {\n  const path = url.replace(/^https?:\\/\\/[^/]+/, \"\").split(\"?\")[0];\n  return JUNK.test(path) || /\\.(?:svg|gif|ico|bmp)$/i.test(path);\n}\n\nfunction attr(attrs: string, name: string): string | undefined {\n  const match = new RegExp(`\\\\b${name}\\\\s*=\\\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\\\s\"'>]+))`, \"i\").exec(attrs);\n  return match ? (match[1] ?? match[2] ?? match[3]) : undefined;\n}\n\nfunction dimension(value: string | undefined): number | undefined {\n  if (!value) return undefined;\n  const n = parseInt(value, 10);\n  return Number.isFinite(n) ? n : undefined;\n}\n\n/**\n * The images a page shows, for a gallery or a review UI: OpenGraph and\n * Twitter card images first, then JSON-LD `image` values, then `<img>`\n * tags including lazy-loaded ones and the largest `srcset` candidate.\n * Tracking pixels, icons, logos, sprites and vector chrome are dropped by\n * size hints and path words, `data:` URIs are ignored, and duplicates are\n * folded. Returns nothing rather than guessing when no `baseUrl` is given\n * and a URL is relative.\n */\nexport function extractImages(html: string, options: ExtractImagesOptions = {}): HarvestedImage[] {\n  const { baseUrl, max = 50, minSize = 100 } = options;\n  const seen = new Set<string>();\n  const images: HarvestedImage[] = [];\n  const add = (raw: string | undefined, extra: Omit<HarvestedImage, \"url\"> = {}) => {\n    if (!raw || images.length >= max) return;\n    const url = toAbsolute(raw, baseUrl);\n    if (!url || seen.has(url) || isJunk(url)) return;\n    if ((extra.width !== undefined && extra.width < minSize) || (extra.height !== undefined && extra.height < minSize)) return;\n    seen.add(url);\n    const image: HarvestedImage = { url };\n    if (extra.alt) image.alt = extra.alt;\n    if (extra.width !== undefined) image.width = extra.width;\n    if (extra.height !== undefined) image.height = extra.height;\n    images.push(image);\n  };\n\n  const meta = extractMeta(html);\n  for (const key of [\"og:image\", \"og:image:secure_url\", \"twitter:image\", \"twitter:image:src\"]) {\n    add(meta[key]);\n  }\n\n  const visit = (value: unknown): void => {\n    if (typeof value === \"string\") add(value);\n    else if (Array.isArray(value)) value.forEach(visit);\n    else if (value && typeof value === \"object\") {\n      const record = value as Record<string, unknown>;\n      if (typeof record.url === \"string\" && (record[\"@type\"] === \"ImageObject\" || \"contentUrl\" in record)) add(record.url);\n      if (typeof record.contentUrl === \"string\") add(record.contentUrl);\n      if (\"image\" in record) visit(record.image);\n      if (\"photo\" in record) visit(record.photo);\n      for (const [key, child] of Object.entries(record)) {\n        if (key !== \"image\" && key !== \"photo\" && child && typeof child === \"object\") visit(child);\n      }\n    }\n  };\n  for (const block of extractJsonLd(html)) visit(block);\n\n  const tags = /<img\\b([^>]*)>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = tags.exec(html)) !== null) {\n    const attrs = match[1];\n    const width = dimension(attr(attrs, \"width\"));\n    const height = dimension(attr(attrs, \"height\"));\n    const alt = attr(attrs, \"alt\")?.trim();\n    const srcset = attr(attrs, \"srcset\") ?? attr(attrs, \"data-srcset\");\n    let candidate = attr(attrs, \"src\") ?? attr(attrs, \"data-src\") ?? attr(attrs, \"data-lazy-src\");\n    if (srcset) {\n      // The widest candidate is the one worth keeping.\n      const best = srcset\n        .split(\",\")\n        .map((entry) => entry.trim().split(/\\s+/))\n        .map(([url, size]) => ({ url, w: size?.endsWith(\"w\") ? parseInt(size, 10) : 0 }))\n        .sort((a, b) => b.w - a.w)[0];\n      if (best?.url) candidate = best.url;\n    }\n    add(candidate, { alt: alt || undefined, width, height });\n  }\n\n  return images;\n}\n\n/**\n * Build a labelled SEMBL source from a page, ready to pass to any coercion or\n * to `sembl()`. For a page that may blow the input budget, prefer\n * {@link htmlSources}, which keeps the structured data in its own source.\n */\nexport function htmlSource(html: string, label?: string, options?: HtmlSourceOptions): Source {\n  const text = pageToText(html, options);\n  return label ? { label, text } : { text };\n}\n\n/**\n * A `preprocess` hook that converts every source's text from HTML, for the\n * case where the sources are pages but you would rather keep the fetch and\n * the coercion apart:\n *\n * ```ts\n * await coerce(pages, { provider, schema, preprocess: preprocessHtml() });\n * ```\n */\nexport function preprocessHtml(options?: HtmlSourceOptions): (source: Source) => Source {\n  return (source) => ({ ...source, text: pageToText(source.text, options) });\n}\n","/**\n * A small, dependency-free HTML-to-text pass tuned for feeding a page to a\n * language model rather than for rendering it.\n *\n * It is regex-based, so it is not a parser: malformed markup degrades to\n * slightly worse text rather than to an error, which is the right trade for\n * scraped input. Structured data the page already carries — JSON-LD blocks,\n * OpenGraph and meta tags, the title — is pulled out separately so it can be\n * placed ahead of the body text, where head-keeping truncation preserves it.\n */\n\n/** Elements whose contents never carry readable text. */\nconst DROP_ELEMENTS = [\"script\", \"style\", \"noscript\", \"template\", \"svg\", \"iframe\", \"head\"];\n\n/** Elements that end a line when they open or close. */\nconst BLOCK_ELEMENTS = [\n  \"address\", \"article\", \"aside\", \"blockquote\", \"dd\", \"details\", \"dialog\", \"div\", \"dl\", \"dt\",\n  \"fieldset\", \"figcaption\", \"figure\", \"footer\", \"form\", \"h1\", \"h2\", \"h3\", \"h4\", \"h5\", \"h6\",\n  \"header\", \"hr\", \"main\", \"nav\", \"ol\", \"p\", \"pre\", \"section\", \"summary\", \"table\", \"tbody\",\n  \"td\", \"tfoot\", \"th\", \"thead\", \"tr\", \"ul\",\n];\n\nconst NAMED_ENTITIES: Record<string, string> = {\n  amp: \"&\", lt: \"<\", gt: \">\", quot: '\"', apos: \"'\", nbsp: \" \", copy: \"©\", reg: \"®\",\n  trade: \"™\", hellip: \"…\", mdash: \"—\", ndash: \"–\", lsquo: \"‘\", rsquo: \"’\", ldquo: \"“\",\n  rdquo: \"”\", bull: \"•\", middot: \"·\", deg: \"°\", euro: \"€\", pound: \"£\", yen: \"¥\", cent: \"¢\",\n  frac12: \"½\", frac14: \"¼\", frac34: \"¾\", times: \"×\", laquo: \"«\", raquo: \"»\",\n};\n\n/** Decode numeric and the common named character references. */\nexport function decodeEntities(text: string): string {\n  return text.replace(/&(#x[0-9a-f]+|#\\d+|[a-z][a-z0-9]*);/gi, (match, ref: string) => {\n    if (ref[0] === \"#\") {\n      const code = ref[1].toLowerCase() === \"x\" ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10);\n      return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match;\n    }\n    return NAMED_ENTITIES[ref.toLowerCase()] ?? match;\n  });\n}\n\n/** Remove an element and everything inside it, for each name given. */\nfunction dropElements(html: string, names: readonly string[]): string {\n  return names.reduce(\n    (acc, name) => acc.replace(new RegExp(`<${name}\\\\b[^>]*>[\\\\s\\\\S]*?</${name}\\\\s*>`, \"gi\"), \" \"),\n    html,\n  );\n}\n\n/**\n * Reduce a page's markup to readable text: comments and non-text elements\n * removed, block boundaries turned into line breaks, list items bulleted,\n * entities decoded, whitespace collapsed.\n */\nexport function htmlToText(html: string): string {\n  let text = html.replace(/<!--[\\s\\S]*?-->/g, \" \");\n  text = dropElements(text, DROP_ELEMENTS);\n  text = text.replace(/<br\\s*\\/?>/gi, \"\\n\");\n  text = text.replace(/<li\\b[^>]*>/gi, \"\\n- \");\n  text = text.replace(new RegExp(`</?(?:${BLOCK_ELEMENTS.join(\"|\")})\\\\b[^>]*>`, \"gi\"), \"\\n\");\n  text = text.replace(/<[^>]+>/g, \" \");\n  text = decodeEntities(text);\n  text = text.replace(/[ \\t\\f\\v ]+/g, \" \");\n  text = text\n    .split(\"\\n\")\n    .map((line) => line.trim())\n    .join(\"\\n\")\n    .replace(/\\n{3,}/g, \"\\n\\n\");\n  return text.trim();\n}\n\n/**\n * Every parseable `<script type=\"application/ld+json\">` block on the page.\n * A block that fails to parse is skipped: it is the page's bug, not ours.\n */\nexport function extractJsonLd(html: string): unknown[] {\n  const blocks: unknown[] = [];\n  const pattern = /<script\\b[^>]*type\\s*=\\s*[\"']?application\\/ld\\+json[\"']?[^>]*>([\\s\\S]*?)<\\/script\\s*>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    try {\n      blocks.push(JSON.parse(match[1].trim()));\n    } catch {\n      // Skip a malformed block rather than lose the rest of the page.\n    }\n  }\n  return blocks;\n}\n\n/** Read one attribute off a tag's attribute string. */\nfunction attribute(attrs: string, name: string): string | undefined {\n  const match = new RegExp(`\\\\b${name}\\\\s*=\\\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\\\s\"'>]+))`, \"i\").exec(attrs);\n  if (!match) return undefined;\n  return decodeEntities(match[1] ?? match[2] ?? match[3] ?? \"\");\n}\n\n/**\n * The page's `<title>` and its `<meta>` tags keyed by `property` or `name`\n * — OpenGraph (`og:*`), Twitter cards, `description`, and so on. Later tags\n * with the same key win, matching how most scrapers read them.\n */\nexport function extractMeta(html: string): Record<string, string> {\n  const meta: Record<string, string> = {};\n  const title = /<title\\b[^>]*>([\\s\\S]*?)<\\/title\\s*>/i.exec(html);\n  if (title) {\n    const text = decodeEntities(title[1]).replace(/\\s+/g, \" \").trim();\n    if (text) meta.title = text;\n  }\n  const pattern = /<meta\\b([^>]*)>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    const attrs = match[1];\n    const key = attribute(attrs, \"property\") ?? attribute(attrs, \"name\");\n    const content = attribute(attrs, \"content\");\n    if (key && content !== undefined && content.trim()) {\n      meta[key.toLowerCase()] = content.trim();\n    }\n  }\n  return meta;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACYA,IAAM,gBAAgB,CAAC,UAAU,SAAS,YAAY,YAAY,OAAO,UAAU,MAAM;AAGzF,IAAM,iBAAiB;AAAA,EACrB;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAAc;AAAA,EAAM;AAAA,EAAW;AAAA,EAAU;AAAA,EAAO;AAAA,EAAM;AAAA,EACrF;AAAA,EAAY;AAAA,EAAc;AAAA,EAAU;AAAA,EAAU;AAAA,EAAQ;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EACpF;AAAA,EAAU;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAM;AAAA,EAAK;AAAA,EAAO;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAChF;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AACtC;AAEA,IAAM,iBAAyC;AAAA,EAC7C,KAAK;AAAA,EAAK,IAAI;AAAA,EAAK,IAAI;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,KAAK;AAAA,EAC7E,OAAO;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAChF,OAAO;AAAA,EAAK,MAAM;AAAA,EAAK,QAAQ;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EAAK,OAAO;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EACrF,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AACxE;AAGO,SAAS,eAAe,MAAsB;AACnD,SAAO,KAAK,QAAQ,yCAAyC,CAAC,OAAO,QAAgB;AACnF,QAAI,IAAI,CAAC,MAAM,KAAK;AAClB,YAAM,OAAO,IAAI,CAAC,EAAE,YAAY,MAAM,MAAM,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE;AAClG,aAAO,OAAO,SAAS,IAAI,KAAK,OAAO,KAAK,QAAQ,UAAW,OAAO,cAAc,IAAI,IAAI;AAAA,IAC9F;AACA,WAAO,eAAe,IAAI,YAAY,CAAC,KAAK;AAAA,EAC9C,CAAC;AACH;AAGA,SAAS,aAAa,MAAc,OAAkC;AACpE,SAAO,MAAM;AAAA,IACX,CAAC,KAAK,SAAS,IAAI,QAAQ,IAAI,OAAO,IAAI,IAAI,wBAAwB,IAAI,SAAS,IAAI,GAAG,GAAG;AAAA,IAC7F;AAAA,EACF;AACF;AAOO,SAAS,WAAW,MAAsB;AAC/C,MAAI,OAAO,KAAK,QAAQ,oBAAoB,GAAG;AAC/C,SAAO,aAAa,MAAM,aAAa;AACvC,SAAO,KAAK,QAAQ,gBAAgB,IAAI;AACxC,SAAO,KAAK,QAAQ,iBAAiB,MAAM;AAC3C,SAAO,KAAK,QAAQ,IAAI,OAAO,SAAS,eAAe,KAAK,GAAG,CAAC,cAAc,IAAI,GAAG,IAAI;AACzF,SAAO,KAAK,QAAQ,YAAY,GAAG;AACnC,SAAO,eAAe,IAAI;AAC1B,SAAO,KAAK,QAAQ,gBAAgB,GAAG;AACvC,SAAO,KACJ,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,KAAK,IAAI,EACT,QAAQ,WAAW,MAAM;AAC5B,SAAO,KAAK,KAAK;AACnB;AAMO,SAAS,cAAc,MAAyB;AACrD,QAAM,SAAoB,CAAC;AAC3B,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,QAAI;AACF,aAAO,KAAK,KAAK,MAAM,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC;AAAA,IACzC,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAGA,SAAS,UAAU,OAAe,MAAkC;AAClE,QAAM,QAAQ,IAAI,OAAO,MAAM,IAAI,iDAAiD,GAAG,EAAE,KAAK,KAAK;AACnG,MAAI,CAAC,MAAO,QAAO;AACnB,SAAO,eAAe,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,EAAE;AAC9D;AAOO,SAAS,YAAY,MAAsC;AAChE,QAAM,OAA+B,CAAC;AACtC,QAAM,QAAQ,wCAAwC,KAAK,IAAI;AAC/D,MAAI,OAAO;AACT,UAAM,OAAO,eAAe,MAAM,CAAC,CAAC,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAChE,QAAI,KAAM,MAAK,QAAQ;AAAA,EACzB;AACA,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,UAAM,QAAQ,MAAM,CAAC;AACrB,UAAM,MAAM,UAAU,OAAO,UAAU,KAAK,UAAU,OAAO,MAAM;AACnE,UAAM,UAAU,UAAU,OAAO,SAAS;AAC1C,QAAI,OAAO,YAAY,UAAa,QAAQ,KAAK,GAAG;AAClD,WAAK,IAAI,YAAY,CAAC,IAAI,QAAQ,KAAK;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;;;AD9FO,SAAS,WAAW,MAAc,UAA6B,CAAC,GAAW;AAChF,QAAM,EAAE,SAAS,MAAM,OAAO,MAAM,OAAO,KAAK,IAAI;AACpD,QAAM,WAAqB,CAAC;AAE5B,MAAI,MAAM;AACR,UAAM,OAAO,YAAY,IAAI;AAC7B,UAAM,QAAQ,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE;AAC3E,QAAI,MAAM,SAAS,EAAG,UAAS,KAAK;AAAA,EAAmB,MAAM,KAAK,IAAI,CAAC,EAAE;AAAA,EAC3E;AAEA,MAAI,QAAQ;AACV,UAAM,SAAS,cAAc,IAAI;AACjC,QAAI,OAAO,SAAS,GAAG;AACrB,eAAS;AAAA,QACP;AAAA,EAA+B,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,MAChF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,MAAM;AACR,UAAM,OAAO,WAAW,IAAI;AAC5B,QAAI,KAAM,UAAS,KAAK;AAAA,EAAe,IAAI,EAAE;AAAA,EAC/C;AAEA,SAAO,SAAS,KAAK,MAAM;AAC7B;AASO,SAAS,YAAY,MAAc,QAAQ,QAAQ,UAA6B,CAAC,GAAa;AACnG,QAAM,EAAE,OAAO,KAAK,IAAI;AACxB,QAAM,aAAa,WAAW,MAAM,EAAE,GAAG,SAAS,MAAM,MAAM,CAAC;AAC/D,QAAM,OAAO,OAAO,WAAW,MAAM,EAAE,MAAM,OAAO,QAAQ,OAAO,MAAM,KAAK,CAAC,IAAI;AACnF,QAAM,UAAoB,CAAC;AAC3B,MAAI,WAAY,SAAQ,KAAK,EAAE,OAAO,GAAG,KAAK,sBAAsB,MAAM,WAAW,CAAC;AACtF,MAAI,KAAM,SAAQ,KAAK,EAAE,OAAO,KAAK,CAAC;AACtC,SAAO;AACT;AAwBA,IAAM,OAAO;AAEb,SAAS,WAAW,KAAa,SAAiD;AAChF,QAAM,UAAU,eAAe,IAAI,KAAK,CAAC;AACzC,MAAI,CAAC,WAAW,QAAQ,WAAW,OAAO,KAAK,QAAQ,WAAW,OAAO,KAAK,QAAQ,WAAW,aAAa,EAAG,QAAO;AACxH,MAAI;AACF,UAAM,WAAW,UAAU,IAAI,IAAI,SAAS,OAAO,IAAI,IAAI,IAAI,OAAO;AACtE,QAAI,SAAS,aAAa,WAAW,SAAS,aAAa,SAAU,QAAO;AAC5E,WAAO,SAAS;AAAA,EAClB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,OAAO,KAAsB;AACpC,QAAM,OAAO,IAAI,QAAQ,qBAAqB,EAAE,EAAE,MAAM,GAAG,EAAE,CAAC;AAC9D,SAAO,KAAK,KAAK,IAAI,KAAK,0BAA0B,KAAK,IAAI;AAC/D;AAEA,SAAS,KAAK,OAAe,MAAkC;AAC7D,QAAM,QAAQ,IAAI,OAAO,MAAM,IAAI,iDAAiD,GAAG,EAAE,KAAK,KAAK;AACnG,SAAO,QAAS,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,MAAM,CAAC,IAAK;AACtD;AAEA,SAAS,UAAU,OAA+C;AAChE,MAAI,CAAC,MAAO,QAAO;AACnB,QAAM,IAAI,SAAS,OAAO,EAAE;AAC5B,SAAO,OAAO,SAAS,CAAC,IAAI,IAAI;AAClC;AAWO,SAAS,cAAc,MAAc,UAAgC,CAAC,GAAqB;AAChG,QAAM,EAAE,SAAS,MAAM,IAAI,UAAU,IAAI,IAAI;AAC7C,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,SAA2B,CAAC;AAClC,QAAM,MAAM,CAAC,KAAyB,QAAqC,CAAC,MAAM;AAChF,QAAI,CAAC,OAAO,OAAO,UAAU,IAAK;AAClC,UAAM,MAAM,WAAW,KAAK,OAAO;AACnC,QAAI,CAAC,OAAO,KAAK,IAAI,GAAG,KAAK,OAAO,GAAG,EAAG;AAC1C,QAAK,MAAM,UAAU,UAAa,MAAM,QAAQ,WAAa,MAAM,WAAW,UAAa,MAAM,SAAS,QAAU;AACpH,SAAK,IAAI,GAAG;AACZ,UAAM,QAAwB,EAAE,IAAI;AACpC,QAAI,MAAM,IAAK,OAAM,MAAM,MAAM;AACjC,QAAI,MAAM,UAAU,OAAW,OAAM,QAAQ,MAAM;AACnD,QAAI,MAAM,WAAW,OAAW,OAAM,SAAS,MAAM;AACrD,WAAO,KAAK,KAAK;AAAA,EACnB;AAEA,QAAM,OAAO,YAAY,IAAI;AAC7B,aAAW,OAAO,CAAC,YAAY,uBAAuB,iBAAiB,mBAAmB,GAAG;AAC3F,QAAI,KAAK,GAAG,CAAC;AAAA,EACf;AAEA,QAAM,QAAQ,CAAC,UAAyB;AACtC,QAAI,OAAO,UAAU,SAAU,KAAI,KAAK;AAAA,aAC/B,MAAM,QAAQ,KAAK,EAAG,OAAM,QAAQ,KAAK;AAAA,aACzC,SAAS,OAAO,UAAU,UAAU;AAC3C,YAAM,SAAS;AACf,UAAI,OAAO,OAAO,QAAQ,aAAa,OAAO,OAAO,MAAM,iBAAiB,gBAAgB,QAAS,KAAI,OAAO,GAAG;AACnH,UAAI,OAAO,OAAO,eAAe,SAAU,KAAI,OAAO,UAAU;AAChE,UAAI,WAAW,OAAQ,OAAM,OAAO,KAAK;AACzC,UAAI,WAAW,OAAQ,OAAM,OAAO,KAAK;AACzC,iBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,MAAM,GAAG;AACjD,YAAI,QAAQ,WAAW,QAAQ,WAAW,SAAS,OAAO,UAAU,SAAU,OAAM,KAAK;AAAA,MAC3F;AAAA,IACF;AAAA,EACF;AACA,aAAW,SAAS,cAAc,IAAI,EAAG,OAAM,KAAK;AAEpD,QAAM,OAAO;AACb,MAAI;AACJ,UAAQ,QAAQ,KAAK,KAAK,IAAI,OAAO,MAAM;AACzC,UAAM,QAAQ,MAAM,CAAC;AACrB,UAAM,QAAQ,UAAU,KAAK,OAAO,OAAO,CAAC;AAC5C,UAAM,SAAS,UAAU,KAAK,OAAO,QAAQ,CAAC;AAC9C,UAAM,MAAM,KAAK,OAAO,KAAK,GAAG,KAAK;AACrC,UAAM,SAAS,KAAK,OAAO,QAAQ,KAAK,KAAK,OAAO,aAAa;AACjE,QAAI,YAAY,KAAK,OAAO,KAAK,KAAK,KAAK,OAAO,UAAU,KAAK,KAAK,OAAO,eAAe;AAC5F,QAAI,QAAQ;AAEV,YAAM,OAAO,OACV,MAAM,GAAG,EACT,IAAI,CAAC,UAAU,MAAM,KAAK,EAAE,MAAM,KAAK,CAAC,EACxC,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,EAAE,KAAK,GAAG,MAAM,SAAS,GAAG,IAAI,SAAS,MAAM,EAAE,IAAI,EAAE,EAAE,EAC/E,KAAK,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AAC9B,UAAI,MAAM,IAAK,aAAY,KAAK;AAAA,IAClC;AACA,QAAI,WAAW,EAAE,KAAK,OAAO,QAAW,OAAO,OAAO,CAAC;AAAA,EACzD;AAEA,SAAO;AACT;AAOO,SAAS,WAAW,MAAc,OAAgB,SAAqC;AAC5F,QAAM,OAAO,WAAW,MAAM,OAAO;AACrC,SAAO,QAAQ,EAAE,OAAO,KAAK,IAAI,EAAE,KAAK;AAC1C;AAWO,SAAS,eAAe,SAAyD;AACtF,SAAO,CAAC,YAAY,EAAE,GAAG,QAAQ,MAAM,WAAW,OAAO,MAAM,OAAO,EAAE;AAC1E;","names":[]}

package/dist/index.d.cts

@@ -49,9 +49,47 @@ interface HtmlSourceOptions {
  * survive.
  */
 declare function pageToText(html: string, options?: HtmlSourceOptions): string;
+/**
+ * A page as two sources: the structured data (title, meta tags, JSON-LD)
+ * and the readable text, each labelled. The structured source is short, so
+ * SEMBL's budget — which cuts long sources first — never touches it; a huge
+ * page loses body text, never its JSON-LD. Pass the result straight to a
+ * coercion. Only the sources that have content are returned.
+ */
+declare function htmlSources(html: string, label?: string, options?: HtmlSourceOptions): Source[];
+/** An image the page shows, with whatever size hints it gave. */
+interface HarvestedImage {
+    url: string;
+    alt?: string;
+    width?: number;
+    height?: number;
+}
+/** Options for {@link extractImages}. */
+interface ExtractImagesOptions {
+    /** Resolves relative URLs. Without it, relative URLs are dropped. */
+    baseUrl?: string;
+    /** Most images to return, in page order with meta images first. Default 50. */
+    max?: number;
+    /**
+     * Drop images whose declared width or height is under this many pixels.
+     * Undeclared sizes are kept. Default 100.
+     */
+    minSize?: number;
+}
+/**
+ * The images a page shows, for a gallery or a review UI: OpenGraph and
+ * Twitter card images first, then JSON-LD `image` values, then `<img>`
+ * tags including lazy-loaded ones and the largest `srcset` candidate.
+ * Tracking pixels, icons, logos, sprites and vector chrome are dropped by
+ * size hints and path words, `data:` URIs are ignored, and duplicates are
+ * folded. Returns nothing rather than guessing when no `baseUrl` is given
+ * and a URL is relative.
+ */
+declare function extractImages(html: string, options?: ExtractImagesOptions): HarvestedImage[];
 /**
  * Build a labelled SEMBL source from a page, ready to pass to any coercion or
- * to `sembl()`.
+ * to `sembl()`. For a page that may blow the input budget, prefer
+ * {@link htmlSources}, which keeps the structured data in its own source.
  */
 declare function htmlSource(html: string, label?: string, options?: HtmlSourceOptions): Source;
 /**
@@ -65,4 +103,4 @@ declare function htmlSource(html: string, label?: string, options?: HtmlSourceOp
  */
 declare function preprocessHtml(options?: HtmlSourceOptions): (source: Source) => Source;
 
-export { type HtmlSourceOptions, decodeEntities, extractJsonLd, extractMeta, htmlSource, htmlToText, pageToText, preprocessHtml };
+export { type ExtractImagesOptions, type HarvestedImage, type HtmlSourceOptions, decodeEntities, extractImages, extractJsonLd, extractMeta, htmlSource, htmlSources, htmlToText, pageToText, preprocessHtml };

package/dist/index.d.ts

@@ -49,9 +49,47 @@ interface HtmlSourceOptions {
  * survive.
  */
 declare function pageToText(html: string, options?: HtmlSourceOptions): string;
+/**
+ * A page as two sources: the structured data (title, meta tags, JSON-LD)
+ * and the readable text, each labelled. The structured source is short, so
+ * SEMBL's budget — which cuts long sources first — never touches it; a huge
+ * page loses body text, never its JSON-LD. Pass the result straight to a
+ * coercion. Only the sources that have content are returned.
+ */
+declare function htmlSources(html: string, label?: string, options?: HtmlSourceOptions): Source[];
+/** An image the page shows, with whatever size hints it gave. */
+interface HarvestedImage {
+    url: string;
+    alt?: string;
+    width?: number;
+    height?: number;
+}
+/** Options for {@link extractImages}. */
+interface ExtractImagesOptions {
+    /** Resolves relative URLs. Without it, relative URLs are dropped. */
+    baseUrl?: string;
+    /** Most images to return, in page order with meta images first. Default 50. */
+    max?: number;
+    /**
+     * Drop images whose declared width or height is under this many pixels.
+     * Undeclared sizes are kept. Default 100.
+     */
+    minSize?: number;
+}
+/**
+ * The images a page shows, for a gallery or a review UI: OpenGraph and
+ * Twitter card images first, then JSON-LD `image` values, then `<img>`
+ * tags including lazy-loaded ones and the largest `srcset` candidate.
+ * Tracking pixels, icons, logos, sprites and vector chrome are dropped by
+ * size hints and path words, `data:` URIs are ignored, and duplicates are
+ * folded. Returns nothing rather than guessing when no `baseUrl` is given
+ * and a URL is relative.
+ */
+declare function extractImages(html: string, options?: ExtractImagesOptions): HarvestedImage[];
 /**
  * Build a labelled SEMBL source from a page, ready to pass to any coercion or
- * to `sembl()`.
+ * to `sembl()`. For a page that may blow the input budget, prefer
+ * {@link htmlSources}, which keeps the structured data in its own source.
  */
 declare function htmlSource(html: string, label?: string, options?: HtmlSourceOptions): Source;
 /**
@@ -65,4 +103,4 @@ declare function htmlSource(html: string, label?: string, options?: HtmlSourceOp
  */
 declare function preprocessHtml(options?: HtmlSourceOptions): (source: Source) => Source;
 
-export { type HtmlSourceOptions, decodeEntities, extractJsonLd, extractMeta, htmlSource, htmlToText, pageToText, preprocessHtml };
+export { type ExtractImagesOptions, type HarvestedImage, type HtmlSourceOptions, decodeEntities, extractImages, extractJsonLd, extractMeta, htmlSource, htmlSources, htmlToText, pageToText, preprocessHtml };

package/dist/index.js

@@ -161,6 +161,92 @@ ${text}`);
   }
   return sections.join("\n\n");
 }
+function htmlSources(html, label = "Page", options = {}) {
+  const { body = true } = options;
+  const structured = pageToText(html, { ...options, body: false });
+  const text = body ? pageToText(html, { meta: false, jsonLd: false, body: true }) : "";
+  const sources = [];
+  if (structured) sources.push({ label: `${label} (structured data)`, text: structured });
+  if (text) sources.push({ label, text });
+  return sources;
+}
+var JUNK = /(?:^|[\/_.-])(?:logo|icon|favicon|sprite|avatar|badge|pixel|tracking|blank|spacer|placeholder|loading|spinner|arrow|flag|emoji|button|banner-ad|ads?)(?:[\/_.-]|$)/i;
+function toAbsolute(url, baseUrl) {
+  const trimmed = decodeEntities(url.trim());
+  if (!trimmed || trimmed.startsWith("data:") || trimmed.startsWith("blob:") || trimmed.startsWith("javascript:")) return void 0;
+  try {
+    const absolute = baseUrl ? new URL(trimmed, baseUrl) : new URL(trimmed);
+    if (absolute.protocol !== "http:" && absolute.protocol !== "https:") return void 0;
+    return absolute.href;
+  } catch {
+    return void 0;
+  }
+}
+function isJunk(url) {
+  const path = url.replace(/^https?:\/\/[^/]+/, "").split("?")[0];
+  return JUNK.test(path) || /\.(?:svg|gif|ico|bmp)$/i.test(path);
+}
+function attr(attrs, name) {
+  const match = new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'>]+))`, "i").exec(attrs);
+  return match ? match[1] ?? match[2] ?? match[3] : void 0;
+}
+function dimension(value) {
+  if (!value) return void 0;
+  const n = parseInt(value, 10);
+  return Number.isFinite(n) ? n : void 0;
+}
+function extractImages(html, options = {}) {
+  const { baseUrl, max = 50, minSize = 100 } = options;
+  const seen = /* @__PURE__ */ new Set();
+  const images = [];
+  const add = (raw, extra = {}) => {
+    if (!raw || images.length >= max) return;
+    const url = toAbsolute(raw, baseUrl);
+    if (!url || seen.has(url) || isJunk(url)) return;
+    if (extra.width !== void 0 && extra.width < minSize || extra.height !== void 0 && extra.height < minSize) return;
+    seen.add(url);
+    const image = { url };
+    if (extra.alt) image.alt = extra.alt;
+    if (extra.width !== void 0) image.width = extra.width;
+    if (extra.height !== void 0) image.height = extra.height;
+    images.push(image);
+  };
+  const meta = extractMeta(html);
+  for (const key of ["og:image", "og:image:secure_url", "twitter:image", "twitter:image:src"]) {
+    add(meta[key]);
+  }
+  const visit = (value) => {
+    if (typeof value === "string") add(value);
+    else if (Array.isArray(value)) value.forEach(visit);
+    else if (value && typeof value === "object") {
+      const record = value;
+      if (typeof record.url === "string" && (record["@type"] === "ImageObject" || "contentUrl" in record)) add(record.url);
+      if (typeof record.contentUrl === "string") add(record.contentUrl);
+      if ("image" in record) visit(record.image);
+      if ("photo" in record) visit(record.photo);
+      for (const [key, child] of Object.entries(record)) {
+        if (key !== "image" && key !== "photo" && child && typeof child === "object") visit(child);
+      }
+    }
+  };
+  for (const block of extractJsonLd(html)) visit(block);
+  const tags = /<img\b([^>]*)>/gi;
+  let match;
+  while ((match = tags.exec(html)) !== null) {
+    const attrs = match[1];
+    const width = dimension(attr(attrs, "width"));
+    const height = dimension(attr(attrs, "height"));
+    const alt = attr(attrs, "alt")?.trim();
+    const srcset = attr(attrs, "srcset") ?? attr(attrs, "data-srcset");
+    let candidate = attr(attrs, "src") ?? attr(attrs, "data-src") ?? attr(attrs, "data-lazy-src");
+    if (srcset) {
+      const best = srcset.split(",").map((entry) => entry.trim().split(/\s+/)).map(([url, size]) => ({ url, w: size?.endsWith("w") ? parseInt(size, 10) : 0 })).sort((a, b) => b.w - a.w)[0];
+      if (best?.url) candidate = best.url;
+    }
+    add(candidate, { alt: alt || void 0, width, height });
+  }
+  return images;
+}
 function htmlSource(html, label, options) {
   const text = pageToText(html, options);
   return label ? { label, text } : { text };
@@ -170,9 +256,11 @@ function preprocessHtml(options) {
 }
 export {
   decodeEntities,
+  extractImages,
   extractJsonLd,
   extractMeta,
   htmlSource,
+  htmlSources,
   htmlToText,
   pageToText,
   preprocessHtml

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/html-to-text.ts","../src/index.ts"],"sourcesContent":["/**\n * A small, dependency-free HTML-to-text pass tuned for feeding a page to a\n * language model rather than for rendering it.\n *\n * It is regex-based, so it is not a parser: malformed markup degrades to\n * slightly worse text rather than to an error, which is the right trade for\n * scraped input. Structured data the page already carries — JSON-LD blocks,\n * OpenGraph and meta tags, the title — is pulled out separately so it can be\n * placed ahead of the body text, where head-keeping truncation preserves it.\n */\n\n/** Elements whose contents never carry readable text. */\nconst DROP_ELEMENTS = [\"script\", \"style\", \"noscript\", \"template\", \"svg\", \"iframe\", \"head\"];\n\n/** Elements that end a line when they open or close. */\nconst BLOCK_ELEMENTS = [\n  \"address\", \"article\", \"aside\", \"blockquote\", \"dd\", \"details\", \"dialog\", \"div\", \"dl\", \"dt\",\n  \"fieldset\", \"figcaption\", \"figure\", \"footer\", \"form\", \"h1\", \"h2\", \"h3\", \"h4\", \"h5\", \"h6\",\n  \"header\", \"hr\", \"main\", \"nav\", \"ol\", \"p\", \"pre\", \"section\", \"summary\", \"table\", \"tbody\",\n  \"td\", \"tfoot\", \"th\", \"thead\", \"tr\", \"ul\",\n];\n\nconst NAMED_ENTITIES: Record<string, string> = {\n  amp: \"&\", lt: \"<\", gt: \">\", quot: '\"', apos: \"'\", nbsp: \" \", copy: \"©\", reg: \"®\",\n  trade: \"™\", hellip: \"…\", mdash: \"—\", ndash: \"–\", lsquo: \"‘\", rsquo: \"’\", ldquo: \"“\",\n  rdquo: \"”\", bull: \"•\", middot: \"·\", deg: \"°\", euro: \"€\", pound: \"£\", yen: \"¥\", cent: \"¢\",\n  frac12: \"½\", frac14: \"¼\", frac34: \"¾\", times: \"×\", laquo: \"«\", raquo: \"»\",\n};\n\n/** Decode numeric and the common named character references. */\nexport function decodeEntities(text: string): string {\n  return text.replace(/&(#x[0-9a-f]+|#\\d+|[a-z][a-z0-9]*);/gi, (match, ref: string) => {\n    if (ref[0] === \"#\") {\n      const code = ref[1].toLowerCase() === \"x\" ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10);\n      return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match;\n    }\n    return NAMED_ENTITIES[ref.toLowerCase()] ?? match;\n  });\n}\n\n/** Remove an element and everything inside it, for each name given. */\nfunction dropElements(html: string, names: readonly string[]): string {\n  return names.reduce(\n    (acc, name) => acc.replace(new RegExp(`<${name}\\\\b[^>]*>[\\\\s\\\\S]*?</${name}\\\\s*>`, \"gi\"), \" \"),\n    html,\n  );\n}\n\n/**\n * Reduce a page's markup to readable text: comments and non-text elements\n * removed, block boundaries turned into line breaks, list items bulleted,\n * entities decoded, whitespace collapsed.\n */\nexport function htmlToText(html: string): string {\n  let text = html.replace(/<!--[\\s\\S]*?-->/g, \" \");\n  text = dropElements(text, DROP_ELEMENTS);\n  text = text.replace(/<br\\s*\\/?>/gi, \"\\n\");\n  text = text.replace(/<li\\b[^>]*>/gi, \"\\n- \");\n  text = text.replace(new RegExp(`</?(?:${BLOCK_ELEMENTS.join(\"|\")})\\\\b[^>]*>`, \"gi\"), \"\\n\");\n  text = text.replace(/<[^>]+>/g, \" \");\n  text = decodeEntities(text);\n  text = text.replace(/[ \\t\\f\\v ]+/g, \" \");\n  text = text\n    .split(\"\\n\")\n    .map((line) => line.trim())\n    .join(\"\\n\")\n    .replace(/\\n{3,}/g, \"\\n\\n\");\n  return text.trim();\n}\n\n/**\n * Every parseable `<script type=\"application/ld+json\">` block on the page.\n * A block that fails to parse is skipped: it is the page's bug, not ours.\n */\nexport function extractJsonLd(html: string): unknown[] {\n  const blocks: unknown[] = [];\n  const pattern = /<script\\b[^>]*type\\s*=\\s*[\"']?application\\/ld\\+json[\"']?[^>]*>([\\s\\S]*?)<\\/script\\s*>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    try {\n      blocks.push(JSON.parse(match[1].trim()));\n    } catch {\n      // Skip a malformed block rather than lose the rest of the page.\n    }\n  }\n  return blocks;\n}\n\n/** Read one attribute off a tag's attribute string. */\nfunction attribute(attrs: string, name: string): string | undefined {\n  const match = new RegExp(`\\\\b${name}\\\\s*=\\\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\\\s\"'>]+))`, \"i\").exec(attrs);\n  if (!match) return undefined;\n  return decodeEntities(match[1] ?? match[2] ?? match[3] ?? \"\");\n}\n\n/**\n * The page's `<title>` and its `<meta>` tags keyed by `property` or `name`\n * — OpenGraph (`og:*`), Twitter cards, `description`, and so on. Later tags\n * with the same key win, matching how most scrapers read them.\n */\nexport function extractMeta(html: string): Record<string, string> {\n  const meta: Record<string, string> = {};\n  const title = /<title\\b[^>]*>([\\s\\S]*?)<\\/title\\s*>/i.exec(html);\n  if (title) {\n    const text = decodeEntities(title[1]).replace(/\\s+/g, \" \").trim();\n    if (text) meta.title = text;\n  }\n  const pattern = /<meta\\b([^>]*)>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    const attrs = match[1];\n    const key = attribute(attrs, \"property\") ?? attribute(attrs, \"name\");\n    const content = attribute(attrs, \"content\");\n    if (key && content !== undefined && content.trim()) {\n      meta[key.toLowerCase()] = content.trim();\n    }\n  }\n  return meta;\n}\n","import type { Source } from \"@sembl/core\";\nimport { extractJsonLd, extractMeta, htmlToText } from \"./html-to-text.js\";\n\nexport { htmlToText, extractJsonLd, extractMeta, decodeEntities } from \"./html-to-text.js\";\n\n/** Options for {@link htmlSource} and {@link pageToText}. */\nexport interface HtmlSourceOptions {\n  /** Include JSON-LD blocks ahead of the body text. Default true. */\n  jsonLd?: boolean;\n  /** Include the title and meta tags ahead of the body text. Default true. */\n  meta?: boolean;\n  /** Include the body text. Default true. */\n  body?: boolean;\n}\n\n/**\n * Render a page as text for extraction.\n *\n * Structured data comes first — the title and meta tags, then any JSON-LD —\n * and the readable body last. That order is deliberate: SEMBL's default\n * truncation keeps the head of a source, so on a page that blows the input\n * budget the parts most likely to hold clean facts are the parts that\n * survive.\n */\nexport function pageToText(html: string, options: HtmlSourceOptions = {}): string {\n  const { jsonLd = true, meta = true, body = true } = options;\n  const sections: string[] = [];\n\n  if (meta) {\n    const tags = extractMeta(html);\n    const lines = Object.entries(tags).map(([key, value]) => `${key}: ${value}`);\n    if (lines.length > 0) sections.push(`Page metadata:\\n${lines.join(\"\\n\")}`);\n  }\n\n  if (jsonLd) {\n    const blocks = extractJsonLd(html);\n    if (blocks.length > 0) {\n      sections.push(\n        `Structured data (JSON-LD):\\n${blocks.map((b) => JSON.stringify(b)).join(\"\\n\")}`,\n      );\n    }\n  }\n\n  if (body) {\n    const text = htmlToText(html);\n    if (text) sections.push(`Page text:\\n${text}`);\n  }\n\n  return sections.join(\"\\n\\n\");\n}\n\n/**\n * Build a labelled SEMBL source from a page, ready to pass to any coercion or\n * to `sembl()`.\n */\nexport function htmlSource(html: string, label?: string, options?: HtmlSourceOptions): Source {\n  const text = pageToText(html, options);\n  return label ? { label, text } : { text };\n}\n\n/**\n * A `preprocess` hook that converts every source's text from HTML, for the\n * case where the sources are pages but you would rather keep the fetch and\n * the coercion apart:\n *\n * ```ts\n * await coerce(pages, { provider, schema, preprocess: preprocessHtml() });\n * ```\n */\nexport function preprocessHtml(options?: HtmlSourceOptions): (source: Source) => Source {\n  return (source) => ({ ...source, text: pageToText(source.text, options) });\n}\n"],"mappings":";AAYA,IAAM,gBAAgB,CAAC,UAAU,SAAS,YAAY,YAAY,OAAO,UAAU,MAAM;AAGzF,IAAM,iBAAiB;AAAA,EACrB;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAAc;AAAA,EAAM;AAAA,EAAW;AAAA,EAAU;AAAA,EAAO;AAAA,EAAM;AAAA,EACrF;AAAA,EAAY;AAAA,EAAc;AAAA,EAAU;AAAA,EAAU;AAAA,EAAQ;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EACpF;AAAA,EAAU;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAM;AAAA,EAAK;AAAA,EAAO;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAChF;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AACtC;AAEA,IAAM,iBAAyC;AAAA,EAC7C,KAAK;AAAA,EAAK,IAAI;AAAA,EAAK,IAAI;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,KAAK;AAAA,EAC7E,OAAO;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAChF,OAAO;AAAA,EAAK,MAAM;AAAA,EAAK,QAAQ;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EAAK,OAAO;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EACrF,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AACxE;AAGO,SAAS,eAAe,MAAsB;AACnD,SAAO,KAAK,QAAQ,yCAAyC,CAAC,OAAO,QAAgB;AACnF,QAAI,IAAI,CAAC,MAAM,KAAK;AAClB,YAAM,OAAO,IAAI,CAAC,EAAE,YAAY,MAAM,MAAM,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE;AAClG,aAAO,OAAO,SAAS,IAAI,KAAK,OAAO,KAAK,QAAQ,UAAW,OAAO,cAAc,IAAI,IAAI;AAAA,IAC9F;AACA,WAAO,eAAe,IAAI,YAAY,CAAC,KAAK;AAAA,EAC9C,CAAC;AACH;AAGA,SAAS,aAAa,MAAc,OAAkC;AACpE,SAAO,MAAM;AAAA,IACX,CAAC,KAAK,SAAS,IAAI,QAAQ,IAAI,OAAO,IAAI,IAAI,wBAAwB,IAAI,SAAS,IAAI,GAAG,GAAG;AAAA,IAC7F;AAAA,EACF;AACF;AAOO,SAAS,WAAW,MAAsB;AAC/C,MAAI,OAAO,KAAK,QAAQ,oBAAoB,GAAG;AAC/C,SAAO,aAAa,MAAM,aAAa;AACvC,SAAO,KAAK,QAAQ,gBAAgB,IAAI;AACxC,SAAO,KAAK,QAAQ,iBAAiB,MAAM;AAC3C,SAAO,KAAK,QAAQ,IAAI,OAAO,SAAS,eAAe,KAAK,GAAG,CAAC,cAAc,IAAI,GAAG,IAAI;AACzF,SAAO,KAAK,QAAQ,YAAY,GAAG;AACnC,SAAO,eAAe,IAAI;AAC1B,SAAO,KAAK,QAAQ,gBAAgB,GAAG;AACvC,SAAO,KACJ,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,KAAK,IAAI,EACT,QAAQ,WAAW,MAAM;AAC5B,SAAO,KAAK,KAAK;AACnB;AAMO,SAAS,cAAc,MAAyB;AACrD,QAAM,SAAoB,CAAC;AAC3B,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,QAAI;AACF,aAAO,KAAK,KAAK,MAAM,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC;AAAA,IACzC,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAGA,SAAS,UAAU,OAAe,MAAkC;AAClE,QAAM,QAAQ,IAAI,OAAO,MAAM,IAAI,iDAAiD,GAAG,EAAE,KAAK,KAAK;AACnG,MAAI,CAAC,MAAO,QAAO;AACnB,SAAO,eAAe,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,EAAE;AAC9D;AAOO,SAAS,YAAY,MAAsC;AAChE,QAAM,OAA+B,CAAC;AACtC,QAAM,QAAQ,wCAAwC,KAAK,IAAI;AAC/D,MAAI,OAAO;AACT,UAAM,OAAO,eAAe,MAAM,CAAC,CAAC,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAChE,QAAI,KAAM,MAAK,QAAQ;AAAA,EACzB;AACA,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,UAAM,QAAQ,MAAM,CAAC;AACrB,UAAM,MAAM,UAAU,OAAO,UAAU,KAAK,UAAU,OAAO,MAAM;AACnE,UAAM,UAAU,UAAU,OAAO,SAAS;AAC1C,QAAI,OAAO,YAAY,UAAa,QAAQ,KAAK,GAAG;AAClD,WAAK,IAAI,YAAY,CAAC,IAAI,QAAQ,KAAK;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;;;AC9FO,SAAS,WAAW,MAAc,UAA6B,CAAC,GAAW;AAChF,QAAM,EAAE,SAAS,MAAM,OAAO,MAAM,OAAO,KAAK,IAAI;AACpD,QAAM,WAAqB,CAAC;AAE5B,MAAI,MAAM;AACR,UAAM,OAAO,YAAY,IAAI;AAC7B,UAAM,QAAQ,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE;AAC3E,QAAI,MAAM,SAAS,EAAG,UAAS,KAAK;AAAA,EAAmB,MAAM,KAAK,IAAI,CAAC,EAAE;AAAA,EAC3E;AAEA,MAAI,QAAQ;AACV,UAAM,SAAS,cAAc,IAAI;AACjC,QAAI,OAAO,SAAS,GAAG;AACrB,eAAS;AAAA,QACP;AAAA,EAA+B,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,MAChF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,MAAM;AACR,UAAM,OAAO,WAAW,IAAI;AAC5B,QAAI,KAAM,UAAS,KAAK;AAAA,EAAe,IAAI,EAAE;AAAA,EAC/C;AAEA,SAAO,SAAS,KAAK,MAAM;AAC7B;AAMO,SAAS,WAAW,MAAc,OAAgB,SAAqC;AAC5F,QAAM,OAAO,WAAW,MAAM,OAAO;AACrC,SAAO,QAAQ,EAAE,OAAO,KAAK,IAAI,EAAE,KAAK;AAC1C;AAWO,SAAS,eAAe,SAAyD;AACtF,SAAO,CAAC,YAAY,EAAE,GAAG,QAAQ,MAAM,WAAW,OAAO,MAAM,OAAO,EAAE;AAC1E;","names":[]}
+{"version":3,"sources":["../src/html-to-text.ts","../src/index.ts"],"sourcesContent":["/**\n * A small, dependency-free HTML-to-text pass tuned for feeding a page to a\n * language model rather than for rendering it.\n *\n * It is regex-based, so it is not a parser: malformed markup degrades to\n * slightly worse text rather than to an error, which is the right trade for\n * scraped input. Structured data the page already carries — JSON-LD blocks,\n * OpenGraph and meta tags, the title — is pulled out separately so it can be\n * placed ahead of the body text, where head-keeping truncation preserves it.\n */\n\n/** Elements whose contents never carry readable text. */\nconst DROP_ELEMENTS = [\"script\", \"style\", \"noscript\", \"template\", \"svg\", \"iframe\", \"head\"];\n\n/** Elements that end a line when they open or close. */\nconst BLOCK_ELEMENTS = [\n  \"address\", \"article\", \"aside\", \"blockquote\", \"dd\", \"details\", \"dialog\", \"div\", \"dl\", \"dt\",\n  \"fieldset\", \"figcaption\", \"figure\", \"footer\", \"form\", \"h1\", \"h2\", \"h3\", \"h4\", \"h5\", \"h6\",\n  \"header\", \"hr\", \"main\", \"nav\", \"ol\", \"p\", \"pre\", \"section\", \"summary\", \"table\", \"tbody\",\n  \"td\", \"tfoot\", \"th\", \"thead\", \"tr\", \"ul\",\n];\n\nconst NAMED_ENTITIES: Record<string, string> = {\n  amp: \"&\", lt: \"<\", gt: \">\", quot: '\"', apos: \"'\", nbsp: \" \", copy: \"©\", reg: \"®\",\n  trade: \"™\", hellip: \"…\", mdash: \"—\", ndash: \"–\", lsquo: \"‘\", rsquo: \"’\", ldquo: \"“\",\n  rdquo: \"”\", bull: \"•\", middot: \"·\", deg: \"°\", euro: \"€\", pound: \"£\", yen: \"¥\", cent: \"¢\",\n  frac12: \"½\", frac14: \"¼\", frac34: \"¾\", times: \"×\", laquo: \"«\", raquo: \"»\",\n};\n\n/** Decode numeric and the common named character references. */\nexport function decodeEntities(text: string): string {\n  return text.replace(/&(#x[0-9a-f]+|#\\d+|[a-z][a-z0-9]*);/gi, (match, ref: string) => {\n    if (ref[0] === \"#\") {\n      const code = ref[1].toLowerCase() === \"x\" ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10);\n      return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match;\n    }\n    return NAMED_ENTITIES[ref.toLowerCase()] ?? match;\n  });\n}\n\n/** Remove an element and everything inside it, for each name given. */\nfunction dropElements(html: string, names: readonly string[]): string {\n  return names.reduce(\n    (acc, name) => acc.replace(new RegExp(`<${name}\\\\b[^>]*>[\\\\s\\\\S]*?</${name}\\\\s*>`, \"gi\"), \" \"),\n    html,\n  );\n}\n\n/**\n * Reduce a page's markup to readable text: comments and non-text elements\n * removed, block boundaries turned into line breaks, list items bulleted,\n * entities decoded, whitespace collapsed.\n */\nexport function htmlToText(html: string): string {\n  let text = html.replace(/<!--[\\s\\S]*?-->/g, \" \");\n  text = dropElements(text, DROP_ELEMENTS);\n  text = text.replace(/<br\\s*\\/?>/gi, \"\\n\");\n  text = text.replace(/<li\\b[^>]*>/gi, \"\\n- \");\n  text = text.replace(new RegExp(`</?(?:${BLOCK_ELEMENTS.join(\"|\")})\\\\b[^>]*>`, \"gi\"), \"\\n\");\n  text = text.replace(/<[^>]+>/g, \" \");\n  text = decodeEntities(text);\n  text = text.replace(/[ \\t\\f\\v ]+/g, \" \");\n  text = text\n    .split(\"\\n\")\n    .map((line) => line.trim())\n    .join(\"\\n\")\n    .replace(/\\n{3,}/g, \"\\n\\n\");\n  return text.trim();\n}\n\n/**\n * Every parseable `<script type=\"application/ld+json\">` block on the page.\n * A block that fails to parse is skipped: it is the page's bug, not ours.\n */\nexport function extractJsonLd(html: string): unknown[] {\n  const blocks: unknown[] = [];\n  const pattern = /<script\\b[^>]*type\\s*=\\s*[\"']?application\\/ld\\+json[\"']?[^>]*>([\\s\\S]*?)<\\/script\\s*>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    try {\n      blocks.push(JSON.parse(match[1].trim()));\n    } catch {\n      // Skip a malformed block rather than lose the rest of the page.\n    }\n  }\n  return blocks;\n}\n\n/** Read one attribute off a tag's attribute string. */\nfunction attribute(attrs: string, name: string): string | undefined {\n  const match = new RegExp(`\\\\b${name}\\\\s*=\\\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\\\s\"'>]+))`, \"i\").exec(attrs);\n  if (!match) return undefined;\n  return decodeEntities(match[1] ?? match[2] ?? match[3] ?? \"\");\n}\n\n/**\n * The page's `<title>` and its `<meta>` tags keyed by `property` or `name`\n * — OpenGraph (`og:*`), Twitter cards, `description`, and so on. Later tags\n * with the same key win, matching how most scrapers read them.\n */\nexport function extractMeta(html: string): Record<string, string> {\n  const meta: Record<string, string> = {};\n  const title = /<title\\b[^>]*>([\\s\\S]*?)<\\/title\\s*>/i.exec(html);\n  if (title) {\n    const text = decodeEntities(title[1]).replace(/\\s+/g, \" \").trim();\n    if (text) meta.title = text;\n  }\n  const pattern = /<meta\\b([^>]*)>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    const attrs = match[1];\n    const key = attribute(attrs, \"property\") ?? attribute(attrs, \"name\");\n    const content = attribute(attrs, \"content\");\n    if (key && content !== undefined && content.trim()) {\n      meta[key.toLowerCase()] = content.trim();\n    }\n  }\n  return meta;\n}\n","import type { Source } from \"@sembl/core\";\nimport { decodeEntities, extractJsonLd, extractMeta, htmlToText } from \"./html-to-text.js\";\n\nexport { htmlToText, extractJsonLd, extractMeta, decodeEntities } from \"./html-to-text.js\";\n\n/** Options for {@link htmlSource} and {@link pageToText}. */\nexport interface HtmlSourceOptions {\n  /** Include JSON-LD blocks ahead of the body text. Default true. */\n  jsonLd?: boolean;\n  /** Include the title and meta tags ahead of the body text. Default true. */\n  meta?: boolean;\n  /** Include the body text. Default true. */\n  body?: boolean;\n}\n\n/**\n * Render a page as text for extraction.\n *\n * Structured data comes first — the title and meta tags, then any JSON-LD —\n * and the readable body last. That order is deliberate: SEMBL's default\n * truncation keeps the head of a source, so on a page that blows the input\n * budget the parts most likely to hold clean facts are the parts that\n * survive.\n */\nexport function pageToText(html: string, options: HtmlSourceOptions = {}): string {\n  const { jsonLd = true, meta = true, body = true } = options;\n  const sections: string[] = [];\n\n  if (meta) {\n    const tags = extractMeta(html);\n    const lines = Object.entries(tags).map(([key, value]) => `${key}: ${value}`);\n    if (lines.length > 0) sections.push(`Page metadata:\\n${lines.join(\"\\n\")}`);\n  }\n\n  if (jsonLd) {\n    const blocks = extractJsonLd(html);\n    if (blocks.length > 0) {\n      sections.push(\n        `Structured data (JSON-LD):\\n${blocks.map((b) => JSON.stringify(b)).join(\"\\n\")}`,\n      );\n    }\n  }\n\n  if (body) {\n    const text = htmlToText(html);\n    if (text) sections.push(`Page text:\\n${text}`);\n  }\n\n  return sections.join(\"\\n\\n\");\n}\n\n/**\n * A page as two sources: the structured data (title, meta tags, JSON-LD)\n * and the readable text, each labelled. The structured source is short, so\n * SEMBL's budget — which cuts long sources first — never touches it; a huge\n * page loses body text, never its JSON-LD. Pass the result straight to a\n * coercion. Only the sources that have content are returned.\n */\nexport function htmlSources(html: string, label = \"Page\", options: HtmlSourceOptions = {}): Source[] {\n  const { body = true } = options;\n  const structured = pageToText(html, { ...options, body: false });\n  const text = body ? pageToText(html, { meta: false, jsonLd: false, body: true }) : \"\";\n  const sources: Source[] = [];\n  if (structured) sources.push({ label: `${label} (structured data)`, text: structured });\n  if (text) sources.push({ label, text });\n  return sources;\n}\n\n/** An image the page shows, with whatever size hints it gave. */\nexport interface HarvestedImage {\n  url: string;\n  alt?: string;\n  width?: number;\n  height?: number;\n}\n\n/** Options for {@link extractImages}. */\nexport interface ExtractImagesOptions {\n  /** Resolves relative URLs. Without it, relative URLs are dropped. */\n  baseUrl?: string;\n  /** Most images to return, in page order with meta images first. Default 50. */\n  max?: number;\n  /**\n   * Drop images whose declared width or height is under this many pixels.\n   * Undeclared sizes are kept. Default 100.\n   */\n  minSize?: number;\n}\n\n/** Path words that mark chrome rather than content. */\nconst JUNK = /(?:^|[\\/_.-])(?:logo|icon|favicon|sprite|avatar|badge|pixel|tracking|blank|spacer|placeholder|loading|spinner|arrow|flag|emoji|button|banner-ad|ads?)(?:[\\/_.-]|$)/i;\n\nfunction toAbsolute(url: string, baseUrl: string | undefined): string | undefined {\n  const trimmed = decodeEntities(url.trim());\n  if (!trimmed || trimmed.startsWith(\"data:\") || trimmed.startsWith(\"blob:\") || trimmed.startsWith(\"javascript:\")) return undefined;\n  try {\n    const absolute = baseUrl ? new URL(trimmed, baseUrl) : new URL(trimmed);\n    if (absolute.protocol !== \"http:\" && absolute.protocol !== \"https:\") return undefined;\n    return absolute.href;\n  } catch {\n    return undefined;\n  }\n}\n\nfunction isJunk(url: string): boolean {\n  const path = url.replace(/^https?:\\/\\/[^/]+/, \"\").split(\"?\")[0];\n  return JUNK.test(path) || /\\.(?:svg|gif|ico|bmp)$/i.test(path);\n}\n\nfunction attr(attrs: string, name: string): string | undefined {\n  const match = new RegExp(`\\\\b${name}\\\\s*=\\\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\\\s\"'>]+))`, \"i\").exec(attrs);\n  return match ? (match[1] ?? match[2] ?? match[3]) : undefined;\n}\n\nfunction dimension(value: string | undefined): number | undefined {\n  if (!value) return undefined;\n  const n = parseInt(value, 10);\n  return Number.isFinite(n) ? n : undefined;\n}\n\n/**\n * The images a page shows, for a gallery or a review UI: OpenGraph and\n * Twitter card images first, then JSON-LD `image` values, then `<img>`\n * tags including lazy-loaded ones and the largest `srcset` candidate.\n * Tracking pixels, icons, logos, sprites and vector chrome are dropped by\n * size hints and path words, `data:` URIs are ignored, and duplicates are\n * folded. Returns nothing rather than guessing when no `baseUrl` is given\n * and a URL is relative.\n */\nexport function extractImages(html: string, options: ExtractImagesOptions = {}): HarvestedImage[] {\n  const { baseUrl, max = 50, minSize = 100 } = options;\n  const seen = new Set<string>();\n  const images: HarvestedImage[] = [];\n  const add = (raw: string | undefined, extra: Omit<HarvestedImage, \"url\"> = {}) => {\n    if (!raw || images.length >= max) return;\n    const url = toAbsolute(raw, baseUrl);\n    if (!url || seen.has(url) || isJunk(url)) return;\n    if ((extra.width !== undefined && extra.width < minSize) || (extra.height !== undefined && extra.height < minSize)) return;\n    seen.add(url);\n    const image: HarvestedImage = { url };\n    if (extra.alt) image.alt = extra.alt;\n    if (extra.width !== undefined) image.width = extra.width;\n    if (extra.height !== undefined) image.height = extra.height;\n    images.push(image);\n  };\n\n  const meta = extractMeta(html);\n  for (const key of [\"og:image\", \"og:image:secure_url\", \"twitter:image\", \"twitter:image:src\"]) {\n    add(meta[key]);\n  }\n\n  const visit = (value: unknown): void => {\n    if (typeof value === \"string\") add(value);\n    else if (Array.isArray(value)) value.forEach(visit);\n    else if (value && typeof value === \"object\") {\n      const record = value as Record<string, unknown>;\n      if (typeof record.url === \"string\" && (record[\"@type\"] === \"ImageObject\" || \"contentUrl\" in record)) add(record.url);\n      if (typeof record.contentUrl === \"string\") add(record.contentUrl);\n      if (\"image\" in record) visit(record.image);\n      if (\"photo\" in record) visit(record.photo);\n      for (const [key, child] of Object.entries(record)) {\n        if (key !== \"image\" && key !== \"photo\" && child && typeof child === \"object\") visit(child);\n      }\n    }\n  };\n  for (const block of extractJsonLd(html)) visit(block);\n\n  const tags = /<img\\b([^>]*)>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = tags.exec(html)) !== null) {\n    const attrs = match[1];\n    const width = dimension(attr(attrs, \"width\"));\n    const height = dimension(attr(attrs, \"height\"));\n    const alt = attr(attrs, \"alt\")?.trim();\n    const srcset = attr(attrs, \"srcset\") ?? attr(attrs, \"data-srcset\");\n    let candidate = attr(attrs, \"src\") ?? attr(attrs, \"data-src\") ?? attr(attrs, \"data-lazy-src\");\n    if (srcset) {\n      // The widest candidate is the one worth keeping.\n      const best = srcset\n        .split(\",\")\n        .map((entry) => entry.trim().split(/\\s+/))\n        .map(([url, size]) => ({ url, w: size?.endsWith(\"w\") ? parseInt(size, 10) : 0 }))\n        .sort((a, b) => b.w - a.w)[0];\n      if (best?.url) candidate = best.url;\n    }\n    add(candidate, { alt: alt || undefined, width, height });\n  }\n\n  return images;\n}\n\n/**\n * Build a labelled SEMBL source from a page, ready to pass to any coercion or\n * to `sembl()`. For a page that may blow the input budget, prefer\n * {@link htmlSources}, which keeps the structured data in its own source.\n */\nexport function htmlSource(html: string, label?: string, options?: HtmlSourceOptions): Source {\n  const text = pageToText(html, options);\n  return label ? { label, text } : { text };\n}\n\n/**\n * A `preprocess` hook that converts every source's text from HTML, for the\n * case where the sources are pages but you would rather keep the fetch and\n * the coercion apart:\n *\n * ```ts\n * await coerce(pages, { provider, schema, preprocess: preprocessHtml() });\n * ```\n */\nexport function preprocessHtml(options?: HtmlSourceOptions): (source: Source) => Source {\n  return (source) => ({ ...source, text: pageToText(source.text, options) });\n}\n"],"mappings":";AAYA,IAAM,gBAAgB,CAAC,UAAU,SAAS,YAAY,YAAY,OAAO,UAAU,MAAM;AAGzF,IAAM,iBAAiB;AAAA,EACrB;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAAc;AAAA,EAAM;AAAA,EAAW;AAAA,EAAU;AAAA,EAAO;AAAA,EAAM;AAAA,EACrF;AAAA,EAAY;AAAA,EAAc;AAAA,EAAU;AAAA,EAAU;AAAA,EAAQ;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EACpF;AAAA,EAAU;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAM;AAAA,EAAK;AAAA,EAAO;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAChF;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AACtC;AAEA,IAAM,iBAAyC;AAAA,EAC7C,KAAK;AAAA,EAAK,IAAI;AAAA,EAAK,IAAI;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,KAAK;AAAA,EAC7E,OAAO;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAChF,OAAO;AAAA,EAAK,MAAM;AAAA,EAAK,QAAQ;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EAAK,OAAO;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EACrF,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AACxE;AAGO,SAAS,eAAe,MAAsB;AACnD,SAAO,KAAK,QAAQ,yCAAyC,CAAC,OAAO,QAAgB;AACnF,QAAI,IAAI,CAAC,MAAM,KAAK;AAClB,YAAM,OAAO,IAAI,CAAC,EAAE,YAAY,MAAM,MAAM,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE;AAClG,aAAO,OAAO,SAAS,IAAI,KAAK,OAAO,KAAK,QAAQ,UAAW,OAAO,cAAc,IAAI,IAAI;AAAA,IAC9F;AACA,WAAO,eAAe,IAAI,YAAY,CAAC,KAAK;AAAA,EAC9C,CAAC;AACH;AAGA,SAAS,aAAa,MAAc,OAAkC;AACpE,SAAO,MAAM;AAAA,IACX,CAAC,KAAK,SAAS,IAAI,QAAQ,IAAI,OAAO,IAAI,IAAI,wBAAwB,IAAI,SAAS,IAAI,GAAG,GAAG;AAAA,IAC7F;AAAA,EACF;AACF;AAOO,SAAS,WAAW,MAAsB;AAC/C,MAAI,OAAO,KAAK,QAAQ,oBAAoB,GAAG;AAC/C,SAAO,aAAa,MAAM,aAAa;AACvC,SAAO,KAAK,QAAQ,gBAAgB,IAAI;AACxC,SAAO,KAAK,QAAQ,iBAAiB,MAAM;AAC3C,SAAO,KAAK,QAAQ,IAAI,OAAO,SAAS,eAAe,KAAK,GAAG,CAAC,cAAc,IAAI,GAAG,IAAI;AACzF,SAAO,KAAK,QAAQ,YAAY,GAAG;AACnC,SAAO,eAAe,IAAI;AAC1B,SAAO,KAAK,QAAQ,gBAAgB,GAAG;AACvC,SAAO,KACJ,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,KAAK,IAAI,EACT,QAAQ,WAAW,MAAM;AAC5B,SAAO,KAAK,KAAK;AACnB;AAMO,SAAS,cAAc,MAAyB;AACrD,QAAM,SAAoB,CAAC;AAC3B,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,QAAI;AACF,aAAO,KAAK,KAAK,MAAM,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC;AAAA,IACzC,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAGA,SAAS,UAAU,OAAe,MAAkC;AAClE,QAAM,QAAQ,IAAI,OAAO,MAAM,IAAI,iDAAiD,GAAG,EAAE,KAAK,KAAK;AACnG,MAAI,CAAC,MAAO,QAAO;AACnB,SAAO,eAAe,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,EAAE;AAC9D;AAOO,SAAS,YAAY,MAAsC;AAChE,QAAM,OAA+B,CAAC;AACtC,QAAM,QAAQ,wCAAwC,KAAK,IAAI;AAC/D,MAAI,OAAO;AACT,UAAM,OAAO,eAAe,MAAM,CAAC,CAAC,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAChE,QAAI,KAAM,MAAK,QAAQ;AAAA,EACzB;AACA,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,UAAM,QAAQ,MAAM,CAAC;AACrB,UAAM,MAAM,UAAU,OAAO,UAAU,KAAK,UAAU,OAAO,MAAM;AACnE,UAAM,UAAU,UAAU,OAAO,SAAS;AAC1C,QAAI,OAAO,YAAY,UAAa,QAAQ,KAAK,GAAG;AAClD,WAAK,IAAI,YAAY,CAAC,IAAI,QAAQ,KAAK;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;;;AC9FO,SAAS,WAAW,MAAc,UAA6B,CAAC,GAAW;AAChF,QAAM,EAAE,SAAS,MAAM,OAAO,MAAM,OAAO,KAAK,IAAI;AACpD,QAAM,WAAqB,CAAC;AAE5B,MAAI,MAAM;AACR,UAAM,OAAO,YAAY,IAAI;AAC7B,UAAM,QAAQ,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE;AAC3E,QAAI,MAAM,SAAS,EAAG,UAAS,KAAK;AAAA,EAAmB,MAAM,KAAK,IAAI,CAAC,EAAE;AAAA,EAC3E;AAEA,MAAI,QAAQ;AACV,UAAM,SAAS,cAAc,IAAI;AACjC,QAAI,OAAO,SAAS,GAAG;AACrB,eAAS;AAAA,QACP;AAAA,EAA+B,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,MAChF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,MAAM;AACR,UAAM,OAAO,WAAW,IAAI;AAC5B,QAAI,KAAM,UAAS,KAAK;AAAA,EAAe,IAAI,EAAE;AAAA,EAC/C;AAEA,SAAO,SAAS,KAAK,MAAM;AAC7B;AASO,SAAS,YAAY,MAAc,QAAQ,QAAQ,UAA6B,CAAC,GAAa;AACnG,QAAM,EAAE,OAAO,KAAK,IAAI;AACxB,QAAM,aAAa,WAAW,MAAM,EAAE,GAAG,SAAS,MAAM,MAAM,CAAC;AAC/D,QAAM,OAAO,OAAO,WAAW,MAAM,EAAE,MAAM,OAAO,QAAQ,OAAO,MAAM,KAAK,CAAC,IAAI;AACnF,QAAM,UAAoB,CAAC;AAC3B,MAAI,WAAY,SAAQ,KAAK,EAAE,OAAO,GAAG,KAAK,sBAAsB,MAAM,WAAW,CAAC;AACtF,MAAI,KAAM,SAAQ,KAAK,EAAE,OAAO,KAAK,CAAC;AACtC,SAAO;AACT;AAwBA,IAAM,OAAO;AAEb,SAAS,WAAW,KAAa,SAAiD;AAChF,QAAM,UAAU,eAAe,IAAI,KAAK,CAAC;AACzC,MAAI,CAAC,WAAW,QAAQ,WAAW,OAAO,KAAK,QAAQ,WAAW,OAAO,KAAK,QAAQ,WAAW,aAAa,EAAG,QAAO;AACxH,MAAI;AACF,UAAM,WAAW,UAAU,IAAI,IAAI,SAAS,OAAO,IAAI,IAAI,IAAI,OAAO;AACtE,QAAI,SAAS,aAAa,WAAW,SAAS,aAAa,SAAU,QAAO;AAC5E,WAAO,SAAS;AAAA,EAClB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,OAAO,KAAsB;AACpC,QAAM,OAAO,IAAI,QAAQ,qBAAqB,EAAE,EAAE,MAAM,GAAG,EAAE,CAAC;AAC9D,SAAO,KAAK,KAAK,IAAI,KAAK,0BAA0B,KAAK,IAAI;AAC/D;AAEA,SAAS,KAAK,OAAe,MAAkC;AAC7D,QAAM,QAAQ,IAAI,OAAO,MAAM,IAAI,iDAAiD,GAAG,EAAE,KAAK,KAAK;AACnG,SAAO,QAAS,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,MAAM,CAAC,IAAK;AACtD;AAEA,SAAS,UAAU,OAA+C;AAChE,MAAI,CAAC,MAAO,QAAO;AACnB,QAAM,IAAI,SAAS,OAAO,EAAE;AAC5B,SAAO,OAAO,SAAS,CAAC,IAAI,IAAI;AAClC;AAWO,SAAS,cAAc,MAAc,UAAgC,CAAC,GAAqB;AAChG,QAAM,EAAE,SAAS,MAAM,IAAI,UAAU,IAAI,IAAI;AAC7C,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,SAA2B,CAAC;AAClC,QAAM,MAAM,CAAC,KAAyB,QAAqC,CAAC,MAAM;AAChF,QAAI,CAAC,OAAO,OAAO,UAAU,IAAK;AAClC,UAAM,MAAM,WAAW,KAAK,OAAO;AACnC,QAAI,CAAC,OAAO,KAAK,IAAI,GAAG,KAAK,OAAO,GAAG,EAAG;AAC1C,QAAK,MAAM,UAAU,UAAa,MAAM,QAAQ,WAAa,MAAM,WAAW,UAAa,MAAM,SAAS,QAAU;AACpH,SAAK,IAAI,GAAG;AACZ,UAAM,QAAwB,EAAE,IAAI;AACpC,QAAI,MAAM,IAAK,OAAM,MAAM,MAAM;AACjC,QAAI,MAAM,UAAU,OAAW,OAAM,QAAQ,MAAM;AACnD,QAAI,MAAM,WAAW,OAAW,OAAM,SAAS,MAAM;AACrD,WAAO,KAAK,KAAK;AAAA,EACnB;AAEA,QAAM,OAAO,YAAY,IAAI;AAC7B,aAAW,OAAO,CAAC,YAAY,uBAAuB,iBAAiB,mBAAmB,GAAG;AAC3F,QAAI,KAAK,GAAG,CAAC;AAAA,EACf;AAEA,QAAM,QAAQ,CAAC,UAAyB;AACtC,QAAI,OAAO,UAAU,SAAU,KAAI,KAAK;AAAA,aAC/B,MAAM,QAAQ,KAAK,EAAG,OAAM,QAAQ,KAAK;AAAA,aACzC,SAAS,OAAO,UAAU,UAAU;AAC3C,YAAM,SAAS;AACf,UAAI,OAAO,OAAO,QAAQ,aAAa,OAAO,OAAO,MAAM,iBAAiB,gBAAgB,QAAS,KAAI,OAAO,GAAG;AACnH,UAAI,OAAO,OAAO,eAAe,SAAU,KAAI,OAAO,UAAU;AAChE,UAAI,WAAW,OAAQ,OAAM,OAAO,KAAK;AACzC,UAAI,WAAW,OAAQ,OAAM,OAAO,KAAK;AACzC,iBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,MAAM,GAAG;AACjD,YAAI,QAAQ,WAAW,QAAQ,WAAW,SAAS,OAAO,UAAU,SAAU,OAAM,KAAK;AAAA,MAC3F;AAAA,IACF;AAAA,EACF;AACA,aAAW,SAAS,cAAc,IAAI,EAAG,OAAM,KAAK;AAEpD,QAAM,OAAO;AACb,MAAI;AACJ,UAAQ,QAAQ,KAAK,KAAK,IAAI,OAAO,MAAM;AACzC,UAAM,QAAQ,MAAM,CAAC;AACrB,UAAM,QAAQ,UAAU,KAAK,OAAO,OAAO,CAAC;AAC5C,UAAM,SAAS,UAAU,KAAK,OAAO,QAAQ,CAAC;AAC9C,UAAM,MAAM,KAAK,OAAO,KAAK,GAAG,KAAK;AACrC,UAAM,SAAS,KAAK,OAAO,QAAQ,KAAK,KAAK,OAAO,aAAa;AACjE,QAAI,YAAY,KAAK,OAAO,KAAK,KAAK,KAAK,OAAO,UAAU,KAAK,KAAK,OAAO,eAAe;AAC5F,QAAI,QAAQ;AAEV,YAAM,OAAO,OACV,MAAM,GAAG,EACT,IAAI,CAAC,UAAU,MAAM,KAAK,EAAE,MAAM,KAAK,CAAC,EACxC,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,EAAE,KAAK,GAAG,MAAM,SAAS,GAAG,IAAI,SAAS,MAAM,EAAE,IAAI,EAAE,EAAE,EAC/E,KAAK,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AAC9B,UAAI,MAAM,IAAK,aAAY,KAAK;AAAA,IAClC;AACA,QAAI,WAAW,EAAE,KAAK,OAAO,QAAW,OAAO,OAAO,CAAC;AAAA,EACzD;AAEA,SAAO;AACT;AAOO,SAAS,WAAW,MAAc,OAAgB,SAAqC;AAC5F,QAAM,OAAO,WAAW,MAAM,OAAO;AACrC,SAAO,QAAQ,EAAE,OAAO,KAAK,IAAI,EAAE,KAAK;AAC1C;AAWO,SAAS,eAAe,SAAyD;AACtF,SAAO,CAAC,YAAY,EAAE,GAAG,QAAQ,MAAM,WAAW,OAAO,MAAM,OAAO,EAAE;AAC1E;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sembl/source-html",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "Turn an HTML page into readable text for SEMBL: JSON-LD and meta tags first, boilerplate stripped.",
   "keywords": [
     "llm",
@@ -52,7 +52,7 @@
     "provenance": true
   },
   "dependencies": {
-    "@sembl/core": "0.3.0"
+    "@sembl/core": "0.5.0"
   },
   "devDependencies": {
     "@types/node": "^25.5.0",