@sembl/source-html 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sembl contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,46 @@
+# @sembl/source-html
+
+Turn an HTML page into text SEMBL can extract from. Zero dependencies.
+
+```sh
+pnpm add @sembl/source-html
+```
+
+```ts
+import { coerce } from "@sembl/core";
+import { htmlSource } from "@sembl/source-html";
+
+const html = await (await fetch(url)).text();
+const listing = await coerce<Listing>(htmlSource(html, "Airbnb listing"), {
+  provider,
+  schema,
+  maxInputChars: 40_000,
+});
+```
+
+`htmlSource` renders the page in three sections, in this order:
+
+1. **Page metadata** — the `<title>` and every `<meta>` tag keyed by
+   `property` or `name` (OpenGraph, Twitter cards, `description`, …).
+2. **Structured data** — each `<script type="application/ld+json">` block that
+   parses, as compact JSON.
+3. **Page text** — the body with scripts, styles, comments and the head
+   removed, block elements as line breaks, list items bulleted, entities
+   decoded and whitespace collapsed.
+
+Structured data goes first on purpose: SEMBL's default truncation keeps the
+head of a source, so when a page blows the input budget the cleanest facts on
+it are the ones that survive.
+
+The pieces are exported on their own — `pageToText`, `htmlToText`,
+`extractJsonLd`, `extractMeta`, `decodeEntities` — and `preprocessHtml()`
+returns a `preprocess` hook for when the sources are pages but you would
+rather keep fetching and coercion apart:
+
+```ts
+await coerce(pages, { provider, schema, preprocess: preprocessHtml() });
+```
+
+The conversion is regex-based rather than a full parser, so malformed markup
+degrades to slightly worse text instead of an error — the right trade for
+scraped input. It makes no network requests; fetching is yours.

package/dist/index.cjs

@@ -0,0 +1,213 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  decodeEntities: () => decodeEntities,
+  extractJsonLd: () => extractJsonLd,
+  extractMeta: () => extractMeta,
+  htmlSource: () => htmlSource,
+  htmlToText: () => htmlToText,
+  pageToText: () => pageToText,
+  preprocessHtml: () => preprocessHtml
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/html-to-text.ts
+var DROP_ELEMENTS = ["script", "style", "noscript", "template", "svg", "iframe", "head"];
+var BLOCK_ELEMENTS = [
+  "address",
+  "article",
+  "aside",
+  "blockquote",
+  "dd",
+  "details",
+  "dialog",
+  "div",
+  "dl",
+  "dt",
+  "fieldset",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "hr",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "section",
+  "summary",
+  "table",
+  "tbody",
+  "td",
+  "tfoot",
+  "th",
+  "thead",
+  "tr",
+  "ul"
+];
+var NAMED_ENTITIES = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'",
+  nbsp: " ",
+  copy: "\xA9",
+  reg: "\xAE",
+  trade: "\u2122",
+  hellip: "\u2026",
+  mdash: "\u2014",
+  ndash: "\u2013",
+  lsquo: "\u2018",
+  rsquo: "\u2019",
+  ldquo: "\u201C",
+  rdquo: "\u201D",
+  bull: "\u2022",
+  middot: "\xB7",
+  deg: "\xB0",
+  euro: "\u20AC",
+  pound: "\xA3",
+  yen: "\xA5",
+  cent: "\xA2",
+  frac12: "\xBD",
+  frac14: "\xBC",
+  frac34: "\xBE",
+  times: "\xD7",
+  laquo: "\xAB",
+  raquo: "\xBB"
+};
+function decodeEntities(text) {
+  return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z][a-z0-9]*);/gi, (match, ref) => {
+    if (ref[0] === "#") {
+      const code = ref[1].toLowerCase() === "x" ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10);
+      return Number.isFinite(code) && code > 0 && code <= 1114111 ? String.fromCodePoint(code) : match;
+    }
+    return NAMED_ENTITIES[ref.toLowerCase()] ?? match;
+  });
+}
+function dropElements(html, names) {
+  return names.reduce(
+    (acc, name) => acc.replace(new RegExp(`<${name}\\b[^>]*>[\\s\\S]*?</${name}\\s*>`, "gi"), " "),
+    html
+  );
+}
+function htmlToText(html) {
+  let text = html.replace(/<!--[\s\S]*?-->/g, " ");
+  text = dropElements(text, DROP_ELEMENTS);
+  text = text.replace(/<br\s*\/?>/gi, "\n");
+  text = text.replace(/<li\b[^>]*>/gi, "\n- ");
+  text = text.replace(new RegExp(`</?(?:${BLOCK_ELEMENTS.join("|")})\\b[^>]*>`, "gi"), "\n");
+  text = text.replace(/<[^>]+>/g, " ");
+  text = decodeEntities(text);
+  text = text.replace(/[ \t\f\v ]+/g, " ");
+  text = text.split("\n").map((line) => line.trim()).join("\n").replace(/\n{3,}/g, "\n\n");
+  return text.trim();
+}
+function extractJsonLd(html) {
+  const blocks = [];
+  const pattern = /<script\b[^>]*type\s*=\s*["']?application\/ld\+json["']?[^>]*>([\s\S]*?)<\/script\s*>/gi;
+  let match;
+  while ((match = pattern.exec(html)) !== null) {
+    try {
+      blocks.push(JSON.parse(match[1].trim()));
+    } catch {
+    }
+  }
+  return blocks;
+}
+function attribute(attrs, name) {
+  const match = new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'>]+))`, "i").exec(attrs);
+  if (!match) return void 0;
+  return decodeEntities(match[1] ?? match[2] ?? match[3] ?? "");
+}
+function extractMeta(html) {
+  const meta = {};
+  const title = /<title\b[^>]*>([\s\S]*?)<\/title\s*>/i.exec(html);
+  if (title) {
+    const text = decodeEntities(title[1]).replace(/\s+/g, " ").trim();
+    if (text) meta.title = text;
+  }
+  const pattern = /<meta\b([^>]*)>/gi;
+  let match;
+  while ((match = pattern.exec(html)) !== null) {
+    const attrs = match[1];
+    const key = attribute(attrs, "property") ?? attribute(attrs, "name");
+    const content = attribute(attrs, "content");
+    if (key && content !== void 0 && content.trim()) {
+      meta[key.toLowerCase()] = content.trim();
+    }
+  }
+  return meta;
+}
+
+// src/index.ts
+function pageToText(html, options = {}) {
+  const { jsonLd = true, meta = true, body = true } = options;
+  const sections = [];
+  if (meta) {
+    const tags = extractMeta(html);
+    const lines = Object.entries(tags).map(([key, value]) => `${key}: ${value}`);
+    if (lines.length > 0) sections.push(`Page metadata:
+${lines.join("\n")}`);
+  }
+  if (jsonLd) {
+    const blocks = extractJsonLd(html);
+    if (blocks.length > 0) {
+      sections.push(
+        `Structured data (JSON-LD):
+${blocks.map((b) => JSON.stringify(b)).join("\n")}`
+      );
+    }
+  }
+  if (body) {
+    const text = htmlToText(html);
+    if (text) sections.push(`Page text:
+${text}`);
+  }
+  return sections.join("\n\n");
+}
+function htmlSource(html, label, options) {
+  const text = pageToText(html, options);
+  return label ? { label, text } : { text };
+}
+function preprocessHtml(options) {
+  return (source) => ({ ...source, text: pageToText(source.text, options) });
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  decodeEntities,
+  extractJsonLd,
+  extractMeta,
+  htmlSource,
+  htmlToText,
+  pageToText,
+  preprocessHtml
+});
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts","../src/html-to-text.ts"],"sourcesContent":["import type { Source } from \"@sembl/core\";\nimport { extractJsonLd, extractMeta, htmlToText } from \"./html-to-text.js\";\n\nexport { htmlToText, extractJsonLd, extractMeta, decodeEntities } from \"./html-to-text.js\";\n\n/** Options for {@link htmlSource} and {@link pageToText}. */\nexport interface HtmlSourceOptions {\n  /** Include JSON-LD blocks ahead of the body text. Default true. */\n  jsonLd?: boolean;\n  /** Include the title and meta tags ahead of the body text. Default true. */\n  meta?: boolean;\n  /** Include the body text. Default true. */\n  body?: boolean;\n}\n\n/**\n * Render a page as text for extraction.\n *\n * Structured data comes first — the title and meta tags, then any JSON-LD —\n * and the readable body last. That order is deliberate: SEMBL's default\n * truncation keeps the head of a source, so on a page that blows the input\n * budget the parts most likely to hold clean facts are the parts that\n * survive.\n */\nexport function pageToText(html: string, options: HtmlSourceOptions = {}): string {\n  const { jsonLd = true, meta = true, body = true } = options;\n  const sections: string[] = [];\n\n  if (meta) {\n    const tags = extractMeta(html);\n    const lines = Object.entries(tags).map(([key, value]) => `${key}: ${value}`);\n    if (lines.length > 0) sections.push(`Page metadata:\\n${lines.join(\"\\n\")}`);\n  }\n\n  if (jsonLd) {\n    const blocks = extractJsonLd(html);\n    if (blocks.length > 0) {\n      sections.push(\n        `Structured data (JSON-LD):\\n${blocks.map((b) => JSON.stringify(b)).join(\"\\n\")}`,\n      );\n    }\n  }\n\n  if (body) {\n    const text = htmlToText(html);\n    if (text) sections.push(`Page text:\\n${text}`);\n  }\n\n  return sections.join(\"\\n\\n\");\n}\n\n/**\n * Build a labelled SEMBL source from a page, ready to pass to any coercion or\n * to `sembl()`.\n */\nexport function htmlSource(html: string, label?: string, options?: HtmlSourceOptions): Source {\n  const text = pageToText(html, options);\n  return label ? { label, text } : { text };\n}\n\n/**\n * A `preprocess` hook that converts every source's text from HTML, for the\n * case where the sources are pages but you would rather keep the fetch and\n * the coercion apart:\n *\n * ```ts\n * await coerce(pages, { provider, schema, preprocess: preprocessHtml() });\n * ```\n */\nexport function preprocessHtml(options?: HtmlSourceOptions): (source: Source) => Source {\n  return (source) => ({ ...source, text: pageToText(source.text, options) });\n}\n","/**\n * A small, dependency-free HTML-to-text pass tuned for feeding a page to a\n * language model rather than for rendering it.\n *\n * It is regex-based, so it is not a parser: malformed markup degrades to\n * slightly worse text rather than to an error, which is the right trade for\n * scraped input. Structured data the page already carries — JSON-LD blocks,\n * OpenGraph and meta tags, the title — is pulled out separately so it can be\n * placed ahead of the body text, where head-keeping truncation preserves it.\n */\n\n/** Elements whose contents never carry readable text. */\nconst DROP_ELEMENTS = [\"script\", \"style\", \"noscript\", \"template\", \"svg\", \"iframe\", \"head\"];\n\n/** Elements that end a line when they open or close. */\nconst BLOCK_ELEMENTS = [\n  \"address\", \"article\", \"aside\", \"blockquote\", \"dd\", \"details\", \"dialog\", \"div\", \"dl\", \"dt\",\n  \"fieldset\", \"figcaption\", \"figure\", \"footer\", \"form\", \"h1\", \"h2\", \"h3\", \"h4\", \"h5\", \"h6\",\n  \"header\", \"hr\", \"main\", \"nav\", \"ol\", \"p\", \"pre\", \"section\", \"summary\", \"table\", \"tbody\",\n  \"td\", \"tfoot\", \"th\", \"thead\", \"tr\", \"ul\",\n];\n\nconst NAMED_ENTITIES: Record<string, string> = {\n  amp: \"&\", lt: \"<\", gt: \">\", quot: '\"', apos: \"'\", nbsp: \" \", copy: \"©\", reg: \"®\",\n  trade: \"™\", hellip: \"…\", mdash: \"—\", ndash: \"–\", lsquo: \"‘\", rsquo: \"’\", ldquo: \"“\",\n  rdquo: \"”\", bull: \"•\", middot: \"·\", deg: \"°\", euro: \"€\", pound: \"£\", yen: \"¥\", cent: \"¢\",\n  frac12: \"½\", frac14: \"¼\", frac34: \"¾\", times: \"×\", laquo: \"«\", raquo: \"»\",\n};\n\n/** Decode numeric and the common named character references. */\nexport function decodeEntities(text: string): string {\n  return text.replace(/&(#x[0-9a-f]+|#\\d+|[a-z][a-z0-9]*);/gi, (match, ref: string) => {\n    if (ref[0] === \"#\") {\n      const code = ref[1].toLowerCase() === \"x\" ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10);\n      return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match;\n    }\n    return NAMED_ENTITIES[ref.toLowerCase()] ?? match;\n  });\n}\n\n/** Remove an element and everything inside it, for each name given. */\nfunction dropElements(html: string, names: readonly string[]): string {\n  return names.reduce(\n    (acc, name) => acc.replace(new RegExp(`<${name}\\\\b[^>]*>[\\\\s\\\\S]*?</${name}\\\\s*>`, \"gi\"), \" \"),\n    html,\n  );\n}\n\n/**\n * Reduce a page's markup to readable text: comments and non-text elements\n * removed, block boundaries turned into line breaks, list items bulleted,\n * entities decoded, whitespace collapsed.\n */\nexport function htmlToText(html: string): string {\n  let text = html.replace(/<!--[\\s\\S]*?-->/g, \" \");\n  text = dropElements(text, DROP_ELEMENTS);\n  text = text.replace(/<br\\s*\\/?>/gi, \"\\n\");\n  text = text.replace(/<li\\b[^>]*>/gi, \"\\n- \");\n  text = text.replace(new RegExp(`</?(?:${BLOCK_ELEMENTS.join(\"|\")})\\\\b[^>]*>`, \"gi\"), \"\\n\");\n  text = text.replace(/<[^>]+>/g, \" \");\n  text = decodeEntities(text);\n  text = text.replace(/[ \\t\\f\\v ]+/g, \" \");\n  text = text\n    .split(\"\\n\")\n    .map((line) => line.trim())\n    .join(\"\\n\")\n    .replace(/\\n{3,}/g, \"\\n\\n\");\n  return text.trim();\n}\n\n/**\n * Every parseable `<script type=\"application/ld+json\">` block on the page.\n * A block that fails to parse is skipped: it is the page's bug, not ours.\n */\nexport function extractJsonLd(html: string): unknown[] {\n  const blocks: unknown[] = [];\n  const pattern = /<script\\b[^>]*type\\s*=\\s*[\"']?application\\/ld\\+json[\"']?[^>]*>([\\s\\S]*?)<\\/script\\s*>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    try {\n      blocks.push(JSON.parse(match[1].trim()));\n    } catch {\n      // Skip a malformed block rather than lose the rest of the page.\n    }\n  }\n  return blocks;\n}\n\n/** Read one attribute off a tag's attribute string. */\nfunction attribute(attrs: string, name: string): string | undefined {\n  const match = new RegExp(`\\\\b${name}\\\\s*=\\\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\\\s\"'>]+))`, \"i\").exec(attrs);\n  if (!match) return undefined;\n  return decodeEntities(match[1] ?? match[2] ?? match[3] ?? \"\");\n}\n\n/**\n * The page's `<title>` and its `<meta>` tags keyed by `property` or `name`\n * — OpenGraph (`og:*`), Twitter cards, `description`, and so on. Later tags\n * with the same key win, matching how most scrapers read them.\n */\nexport function extractMeta(html: string): Record<string, string> {\n  const meta: Record<string, string> = {};\n  const title = /<title\\b[^>]*>([\\s\\S]*?)<\\/title\\s*>/i.exec(html);\n  if (title) {\n    const text = decodeEntities(title[1]).replace(/\\s+/g, \" \").trim();\n    if (text) meta.title = text;\n  }\n  const pattern = /<meta\\b([^>]*)>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    const attrs = match[1];\n    const key = attribute(attrs, \"property\") ?? attribute(attrs, \"name\");\n    const content = attribute(attrs, \"content\");\n    if (key && content !== undefined && content.trim()) {\n      meta[key.toLowerCase()] = content.trim();\n    }\n  }\n  return meta;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACYA,IAAM,gBAAgB,CAAC,UAAU,SAAS,YAAY,YAAY,OAAO,UAAU,MAAM;AAGzF,IAAM,iBAAiB;AAAA,EACrB;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAAc;AAAA,EAAM;AAAA,EAAW;AAAA,EAAU;AAAA,EAAO;AAAA,EAAM;AAAA,EACrF;AAAA,EAAY;AAAA,EAAc;AAAA,EAAU;AAAA,EAAU;AAAA,EAAQ;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EACpF;AAAA,EAAU;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAM;AAAA,EAAK;AAAA,EAAO;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAChF;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AACtC;AAEA,IAAM,iBAAyC;AAAA,EAC7C,KAAK;AAAA,EAAK,IAAI;AAAA,EAAK,IAAI;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,KAAK;AAAA,EAC7E,OAAO;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAChF,OAAO;AAAA,EAAK,MAAM;AAAA,EAAK,QAAQ;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EAAK,OAAO;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EACrF,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AACxE;AAGO,SAAS,eAAe,MAAsB;AACnD,SAAO,KAAK,QAAQ,yCAAyC,CAAC,OAAO,QAAgB;AACnF,QAAI,IAAI,CAAC,MAAM,KAAK;AAClB,YAAM,OAAO,IAAI,CAAC,EAAE,YAAY,MAAM,MAAM,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE;AAClG,aAAO,OAAO,SAAS,IAAI,KAAK,OAAO,KAAK,QAAQ,UAAW,OAAO,cAAc,IAAI,IAAI;AAAA,IAC9F;AACA,WAAO,eAAe,IAAI,YAAY,CAAC,KAAK;AAAA,EAC9C,CAAC;AACH;AAGA,SAAS,aAAa,MAAc,OAAkC;AACpE,SAAO,MAAM;AAAA,IACX,CAAC,KAAK,SAAS,IAAI,QAAQ,IAAI,OAAO,IAAI,IAAI,wBAAwB,IAAI,SAAS,IAAI,GAAG,GAAG;AAAA,IAC7F;AAAA,EACF;AACF;AAOO,SAAS,WAAW,MAAsB;AAC/C,MAAI,OAAO,KAAK,QAAQ,oBAAoB,GAAG;AAC/C,SAAO,aAAa,MAAM,aAAa;AACvC,SAAO,KAAK,QAAQ,gBAAgB,IAAI;AACxC,SAAO,KAAK,QAAQ,iBAAiB,MAAM;AAC3C,SAAO,KAAK,QAAQ,IAAI,OAAO,SAAS,eAAe,KAAK,GAAG,CAAC,cAAc,IAAI,GAAG,IAAI;AACzF,SAAO,KAAK,QAAQ,YAAY,GAAG;AACnC,SAAO,eAAe,IAAI;AAC1B,SAAO,KAAK,QAAQ,gBAAgB,GAAG;AACvC,SAAO,KACJ,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,KAAK,IAAI,EACT,QAAQ,WAAW,MAAM;AAC5B,SAAO,KAAK,KAAK;AACnB;AAMO,SAAS,cAAc,MAAyB;AACrD,QAAM,SAAoB,CAAC;AAC3B,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,QAAI;AACF,aAAO,KAAK,KAAK,MAAM,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC;AAAA,IACzC,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAGA,SAAS,UAAU,OAAe,MAAkC;AAClE,QAAM,QAAQ,IAAI,OAAO,MAAM,IAAI,iDAAiD,GAAG,EAAE,KAAK,KAAK;AACnG,MAAI,CAAC,MAAO,QAAO;AACnB,SAAO,eAAe,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,EAAE;AAC9D;AAOO,SAAS,YAAY,MAAsC;AAChE,QAAM,OAA+B,CAAC;AACtC,QAAM,QAAQ,wCAAwC,KAAK,IAAI;AAC/D,MAAI,OAAO;AACT,UAAM,OAAO,eAAe,MAAM,CAAC,CAAC,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAChE,QAAI,KAAM,MAAK,QAAQ;AAAA,EACzB;AACA,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,UAAM,QAAQ,MAAM,CAAC;AACrB,UAAM,MAAM,UAAU,OAAO,UAAU,KAAK,UAAU,OAAO,MAAM;AACnE,UAAM,UAAU,UAAU,OAAO,SAAS;AAC1C,QAAI,OAAO,YAAY,UAAa,QAAQ,KAAK,GAAG;AAClD,WAAK,IAAI,YAAY,CAAC,IAAI,QAAQ,KAAK;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;;;AD9FO,SAAS,WAAW,MAAc,UAA6B,CAAC,GAAW;AAChF,QAAM,EAAE,SAAS,MAAM,OAAO,MAAM,OAAO,KAAK,IAAI;AACpD,QAAM,WAAqB,CAAC;AAE5B,MAAI,MAAM;AACR,UAAM,OAAO,YAAY,IAAI;AAC7B,UAAM,QAAQ,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE;AAC3E,QAAI,MAAM,SAAS,EAAG,UAAS,KAAK;AAAA,EAAmB,MAAM,KAAK,IAAI,CAAC,EAAE;AAAA,EAC3E;AAEA,MAAI,QAAQ;AACV,UAAM,SAAS,cAAc,IAAI;AACjC,QAAI,OAAO,SAAS,GAAG;AACrB,eAAS;AAAA,QACP;AAAA,EAA+B,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,MAChF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,MAAM;AACR,UAAM,OAAO,WAAW,IAAI;AAC5B,QAAI,KAAM,UAAS,KAAK;AAAA,EAAe,IAAI,EAAE;AAAA,EAC/C;AAEA,SAAO,SAAS,KAAK,MAAM;AAC7B;AAMO,SAAS,WAAW,MAAc,OAAgB,SAAqC;AAC5F,QAAM,OAAO,WAAW,MAAM,OAAO;AACrC,SAAO,QAAQ,EAAE,OAAO,KAAK,IAAI,EAAE,KAAK;AAC1C;AAWO,SAAS,eAAe,SAAyD;AACtF,SAAO,CAAC,YAAY,EAAE,GAAG,QAAQ,MAAM,WAAW,OAAO,MAAM,OAAO,EAAE;AAC1E;","names":[]}

package/dist/index.d.cts

@@ -0,0 +1,68 @@
+import { Source } from '@sembl/core';
+
+/**
+ * A small, dependency-free HTML-to-text pass tuned for feeding a page to a
+ * language model rather than for rendering it.
+ *
+ * It is regex-based, so it is not a parser: malformed markup degrades to
+ * slightly worse text rather than to an error, which is the right trade for
+ * scraped input. Structured data the page already carries — JSON-LD blocks,
+ * OpenGraph and meta tags, the title — is pulled out separately so it can be
+ * placed ahead of the body text, where head-keeping truncation preserves it.
+ */
+/** Decode numeric and the common named character references. */
+declare function decodeEntities(text: string): string;
+/**
+ * Reduce a page's markup to readable text: comments and non-text elements
+ * removed, block boundaries turned into line breaks, list items bulleted,
+ * entities decoded, whitespace collapsed.
+ */
+declare function htmlToText(html: string): string;
+/**
+ * Every parseable `<script type="application/ld+json">` block on the page.
+ * A block that fails to parse is skipped: it is the page's bug, not ours.
+ */
+declare function extractJsonLd(html: string): unknown[];
+/**
+ * The page's `<title>` and its `<meta>` tags keyed by `property` or `name`
+ * — OpenGraph (`og:*`), Twitter cards, `description`, and so on. Later tags
+ * with the same key win, matching how most scrapers read them.
+ */
+declare function extractMeta(html: string): Record<string, string>;
+
+/** Options for {@link htmlSource} and {@link pageToText}. */
+interface HtmlSourceOptions {
+    /** Include JSON-LD blocks ahead of the body text. Default true. */
+    jsonLd?: boolean;
+    /** Include the title and meta tags ahead of the body text. Default true. */
+    meta?: boolean;
+    /** Include the body text. Default true. */
+    body?: boolean;
+}
+/**
+ * Render a page as text for extraction.
+ *
+ * Structured data comes first — the title and meta tags, then any JSON-LD —
+ * and the readable body last. That order is deliberate: SEMBL's default
+ * truncation keeps the head of a source, so on a page that blows the input
+ * budget the parts most likely to hold clean facts are the parts that
+ * survive.
+ */
+declare function pageToText(html: string, options?: HtmlSourceOptions): string;
+/**
+ * Build a labelled SEMBL source from a page, ready to pass to any coercion or
+ * to `sembl()`.
+ */
+declare function htmlSource(html: string, label?: string, options?: HtmlSourceOptions): Source;
+/**
+ * A `preprocess` hook that converts every source's text from HTML, for the
+ * case where the sources are pages but you would rather keep the fetch and
+ * the coercion apart:
+ *
+ * ```ts
+ * await coerce(pages, { provider, schema, preprocess: preprocessHtml() });
+ * ```
+ */
+declare function preprocessHtml(options?: HtmlSourceOptions): (source: Source) => Source;
+
+export { type HtmlSourceOptions, decodeEntities, extractJsonLd, extractMeta, htmlSource, htmlToText, pageToText, preprocessHtml };

package/dist/index.d.ts

@@ -0,0 +1,68 @@
+import { Source } from '@sembl/core';
+
+/**
+ * A small, dependency-free HTML-to-text pass tuned for feeding a page to a
+ * language model rather than for rendering it.
+ *
+ * It is regex-based, so it is not a parser: malformed markup degrades to
+ * slightly worse text rather than to an error, which is the right trade for
+ * scraped input. Structured data the page already carries — JSON-LD blocks,
+ * OpenGraph and meta tags, the title — is pulled out separately so it can be
+ * placed ahead of the body text, where head-keeping truncation preserves it.
+ */
+/** Decode numeric and the common named character references. */
+declare function decodeEntities(text: string): string;
+/**
+ * Reduce a page's markup to readable text: comments and non-text elements
+ * removed, block boundaries turned into line breaks, list items bulleted,
+ * entities decoded, whitespace collapsed.
+ */
+declare function htmlToText(html: string): string;
+/**
+ * Every parseable `<script type="application/ld+json">` block on the page.
+ * A block that fails to parse is skipped: it is the page's bug, not ours.
+ */
+declare function extractJsonLd(html: string): unknown[];
+/**
+ * The page's `<title>` and its `<meta>` tags keyed by `property` or `name`
+ * — OpenGraph (`og:*`), Twitter cards, `description`, and so on. Later tags
+ * with the same key win, matching how most scrapers read them.
+ */
+declare function extractMeta(html: string): Record<string, string>;
+
+/** Options for {@link htmlSource} and {@link pageToText}. */
+interface HtmlSourceOptions {
+    /** Include JSON-LD blocks ahead of the body text. Default true. */
+    jsonLd?: boolean;
+    /** Include the title and meta tags ahead of the body text. Default true. */
+    meta?: boolean;
+    /** Include the body text. Default true. */
+    body?: boolean;
+}
+/**
+ * Render a page as text for extraction.
+ *
+ * Structured data comes first — the title and meta tags, then any JSON-LD —
+ * and the readable body last. That order is deliberate: SEMBL's default
+ * truncation keeps the head of a source, so on a page that blows the input
+ * budget the parts most likely to hold clean facts are the parts that
+ * survive.
+ */
+declare function pageToText(html: string, options?: HtmlSourceOptions): string;
+/**
+ * Build a labelled SEMBL source from a page, ready to pass to any coercion or
+ * to `sembl()`.
+ */
+declare function htmlSource(html: string, label?: string, options?: HtmlSourceOptions): Source;
+/**
+ * A `preprocess` hook that converts every source's text from HTML, for the
+ * case where the sources are pages but you would rather keep the fetch and
+ * the coercion apart:
+ *
+ * ```ts
+ * await coerce(pages, { provider, schema, preprocess: preprocessHtml() });
+ * ```
+ */
+declare function preprocessHtml(options?: HtmlSourceOptions): (source: Source) => Source;
+
+export { type HtmlSourceOptions, decodeEntities, extractJsonLd, extractMeta, htmlSource, htmlToText, pageToText, preprocessHtml };

package/dist/index.js

@@ -0,0 +1,180 @@
+// src/html-to-text.ts
+var DROP_ELEMENTS = ["script", "style", "noscript", "template", "svg", "iframe", "head"];
+var BLOCK_ELEMENTS = [
+  "address",
+  "article",
+  "aside",
+  "blockquote",
+  "dd",
+  "details",
+  "dialog",
+  "div",
+  "dl",
+  "dt",
+  "fieldset",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "hr",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "section",
+  "summary",
+  "table",
+  "tbody",
+  "td",
+  "tfoot",
+  "th",
+  "thead",
+  "tr",
+  "ul"
+];
+var NAMED_ENTITIES = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'",
+  nbsp: " ",
+  copy: "\xA9",
+  reg: "\xAE",
+  trade: "\u2122",
+  hellip: "\u2026",
+  mdash: "\u2014",
+  ndash: "\u2013",
+  lsquo: "\u2018",
+  rsquo: "\u2019",
+  ldquo: "\u201C",
+  rdquo: "\u201D",
+  bull: "\u2022",
+  middot: "\xB7",
+  deg: "\xB0",
+  euro: "\u20AC",
+  pound: "\xA3",
+  yen: "\xA5",
+  cent: "\xA2",
+  frac12: "\xBD",
+  frac14: "\xBC",
+  frac34: "\xBE",
+  times: "\xD7",
+  laquo: "\xAB",
+  raquo: "\xBB"
+};
+function decodeEntities(text) {
+  return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z][a-z0-9]*);/gi, (match, ref) => {
+    if (ref[0] === "#") {
+      const code = ref[1].toLowerCase() === "x" ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10);
+      return Number.isFinite(code) && code > 0 && code <= 1114111 ? String.fromCodePoint(code) : match;
+    }
+    return NAMED_ENTITIES[ref.toLowerCase()] ?? match;
+  });
+}
+function dropElements(html, names) {
+  return names.reduce(
+    (acc, name) => acc.replace(new RegExp(`<${name}\\b[^>]*>[\\s\\S]*?</${name}\\s*>`, "gi"), " "),
+    html
+  );
+}
+function htmlToText(html) {
+  let text = html.replace(/<!--[\s\S]*?-->/g, " ");
+  text = dropElements(text, DROP_ELEMENTS);
+  text = text.replace(/<br\s*\/?>/gi, "\n");
+  text = text.replace(/<li\b[^>]*>/gi, "\n- ");
+  text = text.replace(new RegExp(`</?(?:${BLOCK_ELEMENTS.join("|")})\\b[^>]*>`, "gi"), "\n");
+  text = text.replace(/<[^>]+>/g, " ");
+  text = decodeEntities(text);
+  text = text.replace(/[ \t\f\v ]+/g, " ");
+  text = text.split("\n").map((line) => line.trim()).join("\n").replace(/\n{3,}/g, "\n\n");
+  return text.trim();
+}
+function extractJsonLd(html) {
+  const blocks = [];
+  const pattern = /<script\b[^>]*type\s*=\s*["']?application\/ld\+json["']?[^>]*>([\s\S]*?)<\/script\s*>/gi;
+  let match;
+  while ((match = pattern.exec(html)) !== null) {
+    try {
+      blocks.push(JSON.parse(match[1].trim()));
+    } catch {
+    }
+  }
+  return blocks;
+}
+function attribute(attrs, name) {
+  const match = new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'>]+))`, "i").exec(attrs);
+  if (!match) return void 0;
+  return decodeEntities(match[1] ?? match[2] ?? match[3] ?? "");
+}
+function extractMeta(html) {
+  const meta = {};
+  const title = /<title\b[^>]*>([\s\S]*?)<\/title\s*>/i.exec(html);
+  if (title) {
+    const text = decodeEntities(title[1]).replace(/\s+/g, " ").trim();
+    if (text) meta.title = text;
+  }
+  const pattern = /<meta\b([^>]*)>/gi;
+  let match;
+  while ((match = pattern.exec(html)) !== null) {
+    const attrs = match[1];
+    const key = attribute(attrs, "property") ?? attribute(attrs, "name");
+    const content = attribute(attrs, "content");
+    if (key && content !== void 0 && content.trim()) {
+      meta[key.toLowerCase()] = content.trim();
+    }
+  }
+  return meta;
+}
+
+// src/index.ts
+function pageToText(html, options = {}) {
+  const { jsonLd = true, meta = true, body = true } = options;
+  const sections = [];
+  if (meta) {
+    const tags = extractMeta(html);
+    const lines = Object.entries(tags).map(([key, value]) => `${key}: ${value}`);
+    if (lines.length > 0) sections.push(`Page metadata:
+${lines.join("\n")}`);
+  }
+  if (jsonLd) {
+    const blocks = extractJsonLd(html);
+    if (blocks.length > 0) {
+      sections.push(
+        `Structured data (JSON-LD):
+${blocks.map((b) => JSON.stringify(b)).join("\n")}`
+      );
+    }
+  }
+  if (body) {
+    const text = htmlToText(html);
+    if (text) sections.push(`Page text:
+${text}`);
+  }
+  return sections.join("\n\n");
+}
+function htmlSource(html, label, options) {
+  const text = pageToText(html, options);
+  return label ? { label, text } : { text };
+}
+function preprocessHtml(options) {
+  return (source) => ({ ...source, text: pageToText(source.text, options) });
+}
+export {
+  decodeEntities,
+  extractJsonLd,
+  extractMeta,
+  htmlSource,
+  htmlToText,
+  pageToText,
+  preprocessHtml
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/html-to-text.ts","../src/index.ts"],"sourcesContent":["/**\n * A small, dependency-free HTML-to-text pass tuned for feeding a page to a\n * language model rather than for rendering it.\n *\n * It is regex-based, so it is not a parser: malformed markup degrades to\n * slightly worse text rather than to an error, which is the right trade for\n * scraped input. Structured data the page already carries — JSON-LD blocks,\n * OpenGraph and meta tags, the title — is pulled out separately so it can be\n * placed ahead of the body text, where head-keeping truncation preserves it.\n */\n\n/** Elements whose contents never carry readable text. */\nconst DROP_ELEMENTS = [\"script\", \"style\", \"noscript\", \"template\", \"svg\", \"iframe\", \"head\"];\n\n/** Elements that end a line when they open or close. */\nconst BLOCK_ELEMENTS = [\n  \"address\", \"article\", \"aside\", \"blockquote\", \"dd\", \"details\", \"dialog\", \"div\", \"dl\", \"dt\",\n  \"fieldset\", \"figcaption\", \"figure\", \"footer\", \"form\", \"h1\", \"h2\", \"h3\", \"h4\", \"h5\", \"h6\",\n  \"header\", \"hr\", \"main\", \"nav\", \"ol\", \"p\", \"pre\", \"section\", \"summary\", \"table\", \"tbody\",\n  \"td\", \"tfoot\", \"th\", \"thead\", \"tr\", \"ul\",\n];\n\nconst NAMED_ENTITIES: Record<string, string> = {\n  amp: \"&\", lt: \"<\", gt: \">\", quot: '\"', apos: \"'\", nbsp: \" \", copy: \"©\", reg: \"®\",\n  trade: \"™\", hellip: \"…\", mdash: \"—\", ndash: \"–\", lsquo: \"‘\", rsquo: \"’\", ldquo: \"“\",\n  rdquo: \"”\", bull: \"•\", middot: \"·\", deg: \"°\", euro: \"€\", pound: \"£\", yen: \"¥\", cent: \"¢\",\n  frac12: \"½\", frac14: \"¼\", frac34: \"¾\", times: \"×\", laquo: \"«\", raquo: \"»\",\n};\n\n/** Decode numeric and the common named character references. */\nexport function decodeEntities(text: string): string {\n  return text.replace(/&(#x[0-9a-f]+|#\\d+|[a-z][a-z0-9]*);/gi, (match, ref: string) => {\n    if (ref[0] === \"#\") {\n      const code = ref[1].toLowerCase() === \"x\" ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10);\n      return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match;\n    }\n    return NAMED_ENTITIES[ref.toLowerCase()] ?? match;\n  });\n}\n\n/** Remove an element and everything inside it, for each name given. */\nfunction dropElements(html: string, names: readonly string[]): string {\n  return names.reduce(\n    (acc, name) => acc.replace(new RegExp(`<${name}\\\\b[^>]*>[\\\\s\\\\S]*?</${name}\\\\s*>`, \"gi\"), \" \"),\n    html,\n  );\n}\n\n/**\n * Reduce a page's markup to readable text: comments and non-text elements\n * removed, block boundaries turned into line breaks, list items bulleted,\n * entities decoded, whitespace collapsed.\n */\nexport function htmlToText(html: string): string {\n  let text = html.replace(/<!--[\\s\\S]*?-->/g, \" \");\n  text = dropElements(text, DROP_ELEMENTS);\n  text = text.replace(/<br\\s*\\/?>/gi, \"\\n\");\n  text = text.replace(/<li\\b[^>]*>/gi, \"\\n- \");\n  text = text.replace(new RegExp(`</?(?:${BLOCK_ELEMENTS.join(\"|\")})\\\\b[^>]*>`, \"gi\"), \"\\n\");\n  text = text.replace(/<[^>]+>/g, \" \");\n  text = decodeEntities(text);\n  text = text.replace(/[ \\t\\f\\v ]+/g, \" \");\n  text = text\n    .split(\"\\n\")\n    .map((line) => line.trim())\n    .join(\"\\n\")\n    .replace(/\\n{3,}/g, \"\\n\\n\");\n  return text.trim();\n}\n\n/**\n * Every parseable `<script type=\"application/ld+json\">` block on the page.\n * A block that fails to parse is skipped: it is the page's bug, not ours.\n */\nexport function extractJsonLd(html: string): unknown[] {\n  const blocks: unknown[] = [];\n  const pattern = /<script\\b[^>]*type\\s*=\\s*[\"']?application\\/ld\\+json[\"']?[^>]*>([\\s\\S]*?)<\\/script\\s*>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    try {\n      blocks.push(JSON.parse(match[1].trim()));\n    } catch {\n      // Skip a malformed block rather than lose the rest of the page.\n    }\n  }\n  return blocks;\n}\n\n/** Read one attribute off a tag's attribute string. */\nfunction attribute(attrs: string, name: string): string | undefined {\n  const match = new RegExp(`\\\\b${name}\\\\s*=\\\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\\\s\"'>]+))`, \"i\").exec(attrs);\n  if (!match) return undefined;\n  return decodeEntities(match[1] ?? match[2] ?? match[3] ?? \"\");\n}\n\n/**\n * The page's `<title>` and its `<meta>` tags keyed by `property` or `name`\n * — OpenGraph (`og:*`), Twitter cards, `description`, and so on. Later tags\n * with the same key win, matching how most scrapers read them.\n */\nexport function extractMeta(html: string): Record<string, string> {\n  const meta: Record<string, string> = {};\n  const title = /<title\\b[^>]*>([\\s\\S]*?)<\\/title\\s*>/i.exec(html);\n  if (title) {\n    const text = decodeEntities(title[1]).replace(/\\s+/g, \" \").trim();\n    if (text) meta.title = text;\n  }\n  const pattern = /<meta\\b([^>]*)>/gi;\n  let match: RegExpExecArray | null;\n  while ((match = pattern.exec(html)) !== null) {\n    const attrs = match[1];\n    const key = attribute(attrs, \"property\") ?? attribute(attrs, \"name\");\n    const content = attribute(attrs, \"content\");\n    if (key && content !== undefined && content.trim()) {\n      meta[key.toLowerCase()] = content.trim();\n    }\n  }\n  return meta;\n}\n","import type { Source } from \"@sembl/core\";\nimport { extractJsonLd, extractMeta, htmlToText } from \"./html-to-text.js\";\n\nexport { htmlToText, extractJsonLd, extractMeta, decodeEntities } from \"./html-to-text.js\";\n\n/** Options for {@link htmlSource} and {@link pageToText}. */\nexport interface HtmlSourceOptions {\n  /** Include JSON-LD blocks ahead of the body text. Default true. */\n  jsonLd?: boolean;\n  /** Include the title and meta tags ahead of the body text. Default true. */\n  meta?: boolean;\n  /** Include the body text. Default true. */\n  body?: boolean;\n}\n\n/**\n * Render a page as text for extraction.\n *\n * Structured data comes first — the title and meta tags, then any JSON-LD —\n * and the readable body last. That order is deliberate: SEMBL's default\n * truncation keeps the head of a source, so on a page that blows the input\n * budget the parts most likely to hold clean facts are the parts that\n * survive.\n */\nexport function pageToText(html: string, options: HtmlSourceOptions = {}): string {\n  const { jsonLd = true, meta = true, body = true } = options;\n  const sections: string[] = [];\n\n  if (meta) {\n    const tags = extractMeta(html);\n    const lines = Object.entries(tags).map(([key, value]) => `${key}: ${value}`);\n    if (lines.length > 0) sections.push(`Page metadata:\\n${lines.join(\"\\n\")}`);\n  }\n\n  if (jsonLd) {\n    const blocks = extractJsonLd(html);\n    if (blocks.length > 0) {\n      sections.push(\n        `Structured data (JSON-LD):\\n${blocks.map((b) => JSON.stringify(b)).join(\"\\n\")}`,\n      );\n    }\n  }\n\n  if (body) {\n    const text = htmlToText(html);\n    if (text) sections.push(`Page text:\\n${text}`);\n  }\n\n  return sections.join(\"\\n\\n\");\n}\n\n/**\n * Build a labelled SEMBL source from a page, ready to pass to any coercion or\n * to `sembl()`.\n */\nexport function htmlSource(html: string, label?: string, options?: HtmlSourceOptions): Source {\n  const text = pageToText(html, options);\n  return label ? { label, text } : { text };\n}\n\n/**\n * A `preprocess` hook that converts every source's text from HTML, for the\n * case where the sources are pages but you would rather keep the fetch and\n * the coercion apart:\n *\n * ```ts\n * await coerce(pages, { provider, schema, preprocess: preprocessHtml() });\n * ```\n */\nexport function preprocessHtml(options?: HtmlSourceOptions): (source: Source) => Source {\n  return (source) => ({ ...source, text: pageToText(source.text, options) });\n}\n"],"mappings":";AAYA,IAAM,gBAAgB,CAAC,UAAU,SAAS,YAAY,YAAY,OAAO,UAAU,MAAM;AAGzF,IAAM,iBAAiB;AAAA,EACrB;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAAc;AAAA,EAAM;AAAA,EAAW;AAAA,EAAU;AAAA,EAAO;AAAA,EAAM;AAAA,EACrF;AAAA,EAAY;AAAA,EAAc;AAAA,EAAU;AAAA,EAAU;AAAA,EAAQ;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EACpF;AAAA,EAAU;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAM;AAAA,EAAK;AAAA,EAAO;AAAA,EAAW;AAAA,EAAW;AAAA,EAAS;AAAA,EAChF;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AAAA,EAAS;AAAA,EAAM;AACtC;AAEA,IAAM,iBAAyC;AAAA,EAC7C,KAAK;AAAA,EAAK,IAAI;AAAA,EAAK,IAAI;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,MAAM;AAAA,EAAK,KAAK;AAAA,EAC7E,OAAO;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAChF,OAAO;AAAA,EAAK,MAAM;AAAA,EAAK,QAAQ;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EAAK,OAAO;AAAA,EAAK,KAAK;AAAA,EAAK,MAAM;AAAA,EACrF,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,QAAQ;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AAAA,EAAK,OAAO;AACxE;AAGO,SAAS,eAAe,MAAsB;AACnD,SAAO,KAAK,QAAQ,yCAAyC,CAAC,OAAO,QAAgB;AACnF,QAAI,IAAI,CAAC,MAAM,KAAK;AAClB,YAAM,OAAO,IAAI,CAAC,EAAE,YAAY,MAAM,MAAM,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,EAAE;AAClG,aAAO,OAAO,SAAS,IAAI,KAAK,OAAO,KAAK,QAAQ,UAAW,OAAO,cAAc,IAAI,IAAI;AAAA,IAC9F;AACA,WAAO,eAAe,IAAI,YAAY,CAAC,KAAK;AAAA,EAC9C,CAAC;AACH;AAGA,SAAS,aAAa,MAAc,OAAkC;AACpE,SAAO,MAAM;AAAA,IACX,CAAC,KAAK,SAAS,IAAI,QAAQ,IAAI,OAAO,IAAI,IAAI,wBAAwB,IAAI,SAAS,IAAI,GAAG,GAAG;AAAA,IAC7F;AAAA,EACF;AACF;AAOO,SAAS,WAAW,MAAsB;AAC/C,MAAI,OAAO,KAAK,QAAQ,oBAAoB,GAAG;AAC/C,SAAO,aAAa,MAAM,aAAa;AACvC,SAAO,KAAK,QAAQ,gBAAgB,IAAI;AACxC,SAAO,KAAK,QAAQ,iBAAiB,MAAM;AAC3C,SAAO,KAAK,QAAQ,IAAI,OAAO,SAAS,eAAe,KAAK,GAAG,CAAC,cAAc,IAAI,GAAG,IAAI;AACzF,SAAO,KAAK,QAAQ,YAAY,GAAG;AACnC,SAAO,eAAe,IAAI;AAC1B,SAAO,KAAK,QAAQ,gBAAgB,GAAG;AACvC,SAAO,KACJ,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,KAAK,IAAI,EACT,QAAQ,WAAW,MAAM;AAC5B,SAAO,KAAK,KAAK;AACnB;AAMO,SAAS,cAAc,MAAyB;AACrD,QAAM,SAAoB,CAAC;AAC3B,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,QAAI;AACF,aAAO,KAAK,KAAK,MAAM,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC;AAAA,IACzC,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAGA,SAAS,UAAU,OAAe,MAAkC;AAClE,QAAM,QAAQ,IAAI,OAAO,MAAM,IAAI,iDAAiD,GAAG,EAAE,KAAK,KAAK;AACnG,MAAI,CAAC,MAAO,QAAO;AACnB,SAAO,eAAe,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,EAAE;AAC9D;AAOO,SAAS,YAAY,MAAsC;AAChE,QAAM,OAA+B,CAAC;AACtC,QAAM,QAAQ,wCAAwC,KAAK,IAAI;AAC/D,MAAI,OAAO;AACT,UAAM,OAAO,eAAe,MAAM,CAAC,CAAC,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAChE,QAAI,KAAM,MAAK,QAAQ;AAAA,EACzB;AACA,QAAM,UAAU;AAChB,MAAI;AACJ,UAAQ,QAAQ,QAAQ,KAAK,IAAI,OAAO,MAAM;AAC5C,UAAM,QAAQ,MAAM,CAAC;AACrB,UAAM,MAAM,UAAU,OAAO,UAAU,KAAK,UAAU,OAAO,MAAM;AACnE,UAAM,UAAU,UAAU,OAAO,SAAS;AAC1C,QAAI,OAAO,YAAY,UAAa,QAAQ,KAAK,GAAG;AAClD,WAAK,IAAI,YAAY,CAAC,IAAI,QAAQ,KAAK;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;;;AC9FO,SAAS,WAAW,MAAc,UAA6B,CAAC,GAAW;AAChF,QAAM,EAAE,SAAS,MAAM,OAAO,MAAM,OAAO,KAAK,IAAI;AACpD,QAAM,WAAqB,CAAC;AAE5B,MAAI,MAAM;AACR,UAAM,OAAO,YAAY,IAAI;AAC7B,UAAM,QAAQ,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE;AAC3E,QAAI,MAAM,SAAS,EAAG,UAAS,KAAK;AAAA,EAAmB,MAAM,KAAK,IAAI,CAAC,EAAE;AAAA,EAC3E;AAEA,MAAI,QAAQ;AACV,UAAM,SAAS,cAAc,IAAI;AACjC,QAAI,OAAO,SAAS,GAAG;AACrB,eAAS;AAAA,QACP;AAAA,EAA+B,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,MAChF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,MAAM;AACR,UAAM,OAAO,WAAW,IAAI;AAC5B,QAAI,KAAM,UAAS,KAAK;AAAA,EAAe,IAAI,EAAE;AAAA,EAC/C;AAEA,SAAO,SAAS,KAAK,MAAM;AAC7B;AAMO,SAAS,WAAW,MAAc,OAAgB,SAAqC;AAC5F,QAAM,OAAO,WAAW,MAAM,OAAO;AACrC,SAAO,QAAQ,EAAE,OAAO,KAAK,IAAI,EAAE,KAAK;AAC1C;AAWO,SAAS,eAAe,SAAyD;AACtF,SAAO,CAAC,YAAY,EAAE,GAAG,QAAQ,MAAM,WAAW,OAAO,MAAM,OAAO,EAAE;AAC1E;","names":[]}

package/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "@sembl/source-html",
+  "version": "0.2.0",
+  "description": "Turn an HTML page into readable text for SEMBL: JSON-LD and meta tags first, boilerplate stripped.",
+  "keywords": [
+    "llm",
+    "extraction",
+    "html",
+    "json-ld",
+    "opengraph",
+    "scraping"
+  ],
+  "license": "MIT",
+  "author": "Sembl contributors",
+  "homepage": "https://github.com/nickrunner/sembl#readme",
+  "bugs": {
+    "url": "https://github.com/nickrunner/sembl/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/nickrunner/sembl.git",
+    "directory": "packages/source-html"
+  },
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "sideEffects": false,
+  "engines": {
+    "node": ">=20"
+  },
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "dependencies": {
+    "@sembl/core": "0.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "tsup": "^8.0.0",
+    "typescript": "^5.5.0"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch"
+  }
+}