page-markdown 0.1.4 → 0.2.0

package/dist/types/refine.d.ts

@@ -0,0 +1,31 @@
+/** The text of serialized HTML: tags to spaces, the entities serializers write decoded. */
+export function htmlText(html: any): string;
+/**
+ * Many sites state their summary as both the meta description and the first
+ * paragraph. Extractors can drop that paragraph when it sits in a section
+ * without a heading (MDN). If the description matches a paragraph in the page
+ * that the extracted HTML lacks, put that paragraph first.
+ *
+ * @param {string} html extracted HTML
+ * @param {Document} doc standardized clone
+ * @param {string} description meta description
+ * @returns {{ html: string, recovered: boolean }}
+ */
+export function recoverLeadParagraph(html: string, doc: Document, description: string): {
+    html: string;
+    recovered: boolean;
+};
+/**
+ * The heading for the capture: the page's own H1 when the document title or
+ * og:title contains it (and it is not just the site name); otherwise og:title;
+ * otherwise the title without a trailing site-name segment.
+ *
+ * @param {Document} doc standardized clone
+ * @param {{ documentTitle?: string, title?: string, siteName?: string, extractedTitle?: string }} [meta]
+ */
+export function chooseHeadline(doc: Document, meta?: {
+    documentTitle?: string;
+    title?: string;
+    siteName?: string;
+    extractedTitle?: string;
+}): string;

package/dist/types/sanitize.d.ts

@@ -0,0 +1,28 @@
+/** @param {any} node */
+export function isAdvertisementElement(node: any): boolean;
+/**
+ * Creates an empty HTML document unconnected to the page: no scripts run and
+ * no custom elements upgrade in it.
+ * @param {Document} doc
+ */
+export function createInertDocument(doc: Document): Document;
+/**
+ * Clones `root` (a Document or Element) into a new inert document.
+ * Open shadow roots replace their host's light children (native slots supply
+ * assigned children). Limits apply only to shadow content, per host in the
+ * page's own tree, so slotted page content is not charged: docs sites put a
+ * whole article in one host's slot. A host over the limit keeps what fits and
+ * is counted as an omission.
+ *
+ * @param {Document|Element} root
+ * @param {import("./types.js").SanitizeOptions} [options]
+ * @returns {{ document: Document, omissions: { shadowHosts: number } }}
+ */
+export function sanitizeToInertDocument(root: Document | Element, options?: import("./types.js").SanitizeOptions): {
+    document: Document;
+    omissions: {
+        shadowHosts: number;
+    };
+};
+/** @type {Readonly<Required<import("./types.js").SanitizeOptions>>} */
+export const DEFAULT_SANITIZE_OPTIONS: Readonly<Required<import("./types.js").SanitizeOptions>>;

package/dist/types/standardize.d.ts

@@ -0,0 +1,16 @@
+/**
+ * @param {Document} doc the inert clone (mutated in place)
+ * @param {{ url?: string }} [options]
+ */
+export function standardizeDocument(doc: Document, options?: {
+    url?: string;
+}): {
+    lazyImagesResolved: number;
+    inlineImagesDropped: number;
+    scriptLinksUnwrapped: number;
+    emptyLinksRemoved: number;
+    preWrapped: number;
+    headingWidgetsRemoved: number;
+    headingsUnwrapped: number;
+    omittedTabs: string[];
+};

package/dist/types/types.d.ts

@@ -0,0 +1,167 @@
+/// <reference lib="dom" preserve="true" />
+export type OutputMode = "full" | "no_links" | "text_only";
+export type SanitizeOptions = {
+    stripScripts?: boolean | undefined;
+    stripStyles?: boolean | undefined;
+    stripNoscript?: boolean | undefined;
+    stripMeta?: boolean | undefined;
+    stripLinkTags?: boolean | undefined;
+    stripAds?: boolean | undefined;
+    /**
+     * app-specific overlays or widgets to drop, e.g. "[data-app-overlay]"
+     */
+    extraStripSelectors?: readonly string[] | undefined;
+    shadowNodeLimit?: number | undefined;
+    shadowCharLimit?: number | undefined;
+    shadowDepthLimit?: number | undefined;
+};
+export type ReadabilityOptions = NonNullable<ConstructorParameters<typeof import("@mozilla/readability").Readability<string>>[1]>;
+export type Extraction = import("./extract.js").Extraction;
+export type Extractor = import("./extract.js").Extractor;
+export type CollectedMetadata = ReturnType<typeof import("./metadata.js").collectMetadata>;
+export type ExtractedMetadata = {
+    /**
+     * the heading chosen for the capture
+     */
+    headline?: string | undefined;
+    /**
+     * the title the extractor reported
+     */
+    extractedTitle?: string | undefined;
+    /**
+     * the byline the extractor reported
+     */
+    byline?: string | undefined;
+};
+export type PageMarkdownMetadata = CollectedMetadata & ExtractedMetadata;
+export type PageMarkdownExtraction = {
+    method: Extraction["method"];
+    /**
+     * why extraction fell back to the full page or the main content
+     */
+    fallbackReason?: string | undefined;
+    /**
+     * a dropped lead paragraph was restored
+     */
+    leadRecovered?: true | undefined;
+    /**
+     * how many facts `pageFacts` appended
+     */
+    pageFacts?: number | undefined;
+};
+export type HiddenRunKind = "tag" | "variation_selector" | "invisible_operator" | "reserved_tag_plane";
+/**
+ * A run as text-sanctifier's findHiddenRuns reports it.
+ */
+export type HiddenRun = {
+    kind: HiddenRunKind;
+    /**
+     * UTF-16 index of the first character
+     */
+    offset: number;
+    /**
+     * UTF-16 index just past the last character
+     */
+    end: number;
+    /**
+     * code points in the run
+     */
+    count: number;
+    /**
+     * 1-based line of the first character
+     */
+    line: number;
+    /**
+     * 1-based column of the first character, in UTF-16 units
+     */
+    column: number;
+    /**
+     * tag runs only: the ASCII the tags spell
+     */
+    decoded?: string | undefined;
+    /**
+     * tag runs only: true when `decoded` stops short of the run
+     */
+    decodedTruncated?: boolean | undefined;
+};
+export type HiddenTextOmission = {
+    kind: `hidden_${HiddenRunKind}`;
+    /**
+     * runs removed from `markdown`
+     */
+    count: number;
+    /**
+     * code points in those runs
+     */
+    chars: number;
+    note: string;
+};
+/**
+ * Check `kind` before reading `chars`: only hidden-text omissions have it.
+ */
+export type PageMarkdownOmission = {
+    kind: "shadow_content";
+    count: number;
+    note: string;
+} | {
+    kind: "json_line";
+    count: number;
+    note: string;
+} | {
+    kind: "inactive_tabs";
+    count: number;
+    note: string;
+} | {
+    kind: "comments";
+    count: number;
+    note: string;
+} | HiddenTextOmission;
+export type StandardizationStats = ReturnType<typeof import("./standardize.js").standardizeDocument>;
+export type PageMarkdownStats = {
+    mode: OutputMode;
+    extractedHtmlChars: number;
+    convertedChars: number;
+    markdownChars: number;
+    /**
+     * the page's text at capture time: a capture is a
+     * snapshot of the DOM, so a caller can re-capture until this number settles
+     */
+    pageTextChars: number;
+    standardization: StandardizationStats;
+};
+export type PageMarkdownResult = {
+    /**
+     * the package version that made this result
+     */
+    version: string;
+    markdown: string;
+    metadata: PageMarkdownMetadata;
+    extraction: PageMarkdownExtraction;
+    omissions: PageMarkdownOmission[];
+    warnings: string[];
+    stats: PageMarkdownStats;
+    /**
+     * present only with `includeFullPage`
+     */
+    fullPageMarkdown?: string | undefined;
+};
+export type PageFact = {
+    label: string;
+    value: string;
+};
+export type TurndownFilter = string | string[] | ((node: HTMLElement, options: Record<string, unknown>) => boolean);
+/**
+ * Turndown's public methods; Turndown ships no types of its own.
+ */
+export type TurndownInstance = {
+    turndown: (input: string | Node) => string;
+    use: (plugin: TurndownPlugin | TurndownPlugin[]) => TurndownInstance;
+    addRule: (key: string, rule: {
+        filter: TurndownFilter;
+        replacement: (content: string, node: Node, options: Record<string, unknown>) => string;
+    }) => TurndownInstance;
+    keep: (filter: TurndownFilter) => TurndownInstance;
+    remove: (filter: TurndownFilter) => TurndownInstance;
+    escape: (text: string) => string;
+};
+export type TurndownPlugin = (service: TurndownInstance) => void;

package/package.json

@@ -1,13 +1,18 @@
 {
   "name": "page-markdown",
-  "version": "0.1.4",
+  "version": "0.2.0",
   "description": "Faithful page-to-Markdown for humans and agents: sanitize, extract the readable article, convert with GFM, and clean without losing content.",
   "type": "module",
   "exports": {
-    ".": "./src/index.js"
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "default": "./src/index.js"
+    }
   },
+  "types": "./dist/types/index.d.ts",
   "files": [
     "src",
+    "dist/types",
     "LICENSE",
     "ADDITIONAL_TERMS.md",
     "THIRD_PARTY_NOTICES.md",
@@ -22,9 +27,15 @@
   },
   "scripts": {
     "test": "node --test \"test/**/*.test.mjs\"",
+    "build:types": "tsc -p tsconfig.json",
+    "typecheck": "npm run build:types && node test/types/real-output.mjs && tsc -p test/types && tsc -p test/types/linkedom",
     "size": "node bench/size.mjs",
-    "ci": "npm test && npm run size",
-    "bench": "node bench/compare.mjs"
+    "test:browser": "node test/browser/run.mjs",
+    "ci": "npm test && npm run typecheck && npm run size",
+    "prepack": "npm run build:types",
+    "prepublishOnly": "npm run ci",
+    "bench": "node bench/compare.mjs",
+    "bench:write": "node bench/compare.mjs --write"
   },
   "keywords": [
     "markdown",
@@ -51,6 +62,7 @@
   },
   "devDependencies": {
     "esbuild": "0.27.0",
-    "linkedom": "0.18.12"
+    "linkedom": "0.18.12",
+    "typescript": "5.9.3"
   }
 }

package/src/clean.js

@@ -13,6 +13,7 @@ export function mapOutsideFences(markdown, transform) {
   const blocks = [];
   let lines = [];
   let fence = null;
+  let indent = 0;
   const flush = (isCode) => {
     if (!lines.length) return;
     const text = lines.join("\n");
@@ -20,12 +21,14 @@ export function mapOutsideFences(markdown, transform) {
     lines = [];
   };
   for (const line of String(markdown).split("\n")) {
-    const match = line.match(/^ {0,3}(`{3,}|~{3,})(.*)$/);
-    if (!fence && match && !(match[1][0] === "`" && match[2].includes("`"))) {
+    // Any indent or quote prefix: Turndown nests code in lists and blockquotes.
+    const match = line.match(/^([ >]*)(`{3,}|~{3,})(.*)$/);
+    if (!fence && match && !(match[2][0] === "`" && match[3].includes("`"))) {
       flush(false);
-      fence = match[1];
+      fence = match[2];
+      indent = match[1].length;
       lines.push(line);
-    } else if (fence && match && match[1][0] === fence[0] && match[1].length >= fence.length && !match[2].trim()) {
+    } else if (fence && match && match[1].length <= indent + 3 && match[2][0] === fence[0] && match[2].length >= fence.length && !match[3].trim()) {
       lines.push(line);
       flush(true);
       fence = null;
@@ -81,8 +84,13 @@ export function cleanMarkdown(markdown) {
 // Link text may hold escaped brackets and whole images: [![alt](img)](href)
 // keeps the image, not "![alt" pointing at href.
 const LINK_TARGET = String.raw`\((?:[^()\s]|\([^()]*\))+(?:\s+"[^"]*")?\)`;
-const INLINE_LINK = new RegExp(String.raw`(!?)\[((?:\\.|!\[[^\]]*\]${LINK_TARGET}|[^\[\]\\])*)\]${LINK_TARGET}`, "g");
-const stripInlineLinks = (md) => md.replace(INLINE_LINK, (all, bang, text) => (bang ? all : text));
+const INLINE_LINK = new RegExp(String.raw`(!?)\[((?:\\.|!\[[^\]]*\]${LINK_TARGET}|[^\[\]\\])*)\]${LINK_TARGET}`);
+// Replaces matches of `pattern`, leaving inline code spans as they are.
+const replaceOutsideCode = (md, pattern, replace) => md.replace(
+  new RegExp(`\`+[^\`\\n]*\`+|${pattern.source}`, "g"),
+  (all, ...groups) => (all[0] === "`" ? all : replace(all, ...groups)),
+);
+const stripInlineLinks = (md) => replaceOutsideCode(md, INLINE_LINK, (all, bang, text) => (bang ? all : text));
 const dropReferenceDefs = (md) => md.replace(/^\s{0,3}\[[^\]]+]:\s+\S+.*$/gm, "");
 const stripStructure = (md) => md
   .replace(/^\s{0,3}#{1,6}\s+/gm, "")
@@ -90,14 +98,18 @@ const stripStructure = (md) => md
   .replace(/^\s*\d+\.\s+/gm, "")
   .replace(/^\s{0,3}>\s?/gm, "")
   .replace(/^\s*([-_*])\s*(?:\1\s*){2,}$/gm, "");
-const stripEmphasis = (md) => md.replace(/(\*\*|__)(.+?)\1/g, "$2").replace(/(?<![\w*])([*_])(?!\s)(.+?)(?<!\s)\1(?![\w*])/g, "$2");
-const stripImages = (md) => md.replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1");
+const stripEmphasis = (md) => replaceOutsideCode(
+  replaceOutsideCode(md, /(\*\*|__)(.+?)\1/, (all, mark, text) => text),
+  /(?<![\w*])([*_])(?!\s)(.+?)(?<!\s)\1(?![\w*])/,
+  (all, mark, text) => text,
+);
+const stripImages = (md) => replaceOutsideCode(md, /!\[([^\]]*)\]\([^)]*\)/, (all, alt) => alt);
 
 /**
  * Output modes. "no_links" keeps link text. "text_only" also drops headings,
  * list markers, emphasis and images. Both leave fenced code verbatim.
  * @param {string} markdown
- * @param {"full"|"no_links"|"text_only"} mode
+ * @param {import("./types.js").OutputMode} [mode]
  */
 export function applyMode(markdown, mode = "full") {
   if (mode === "no_links") return mapOutsideFences(markdown, stripInlineLinks);

package/src/convert.js

@@ -14,6 +14,15 @@ export const TURNDOWN_OPTIONS = Object.freeze({
 });
 
 function gfmInlineRules(turndown) {
+  // Link text is one line: a link around blocks would otherwise split across lines.
+  turndown.addRule("pmInlineLink", {
+    filter: (node) => node.nodeName === "A" && Boolean(node.getAttribute("href")),
+    replacement(content, node) {
+      const href = node.getAttribute("href").replace(/([()])/g, "\\$1");
+      const title = (node.getAttribute("title") || "").replace(/\s*\n+\s*/g, " ").replace(/"/g, '\\"');
+      return `[${content.trim().replace(/\s*\n+\s*/g, " ")}](${href}${title ? ` "${title}"` : ""})`;
+    },
+  });
   turndown.addRule("pmStrikethrough", {
     filter: ["del", "s", "strike"],
     replacement: (content) => `~~${content}~~`,
@@ -65,25 +74,74 @@ function spanOf(cell) {
   return Number.isFinite(span) && span > 1 ? Math.min(span, 50) : 1;
 }
 
-// Layout tables (nesting, a single cell, a single column) read better as
-// blocks than as a grid.
-function isLayoutTable(table) {
-  if (table.querySelector?.("table")) return true;
-  const rows = tableRows(table);
-  if (!rows.length) return true;
-  const widest = Math.max(...rows.map((row) => rowCells(row).reduce((sum, cell) => sum + spanOf(cell), 0)));
-  return widest <= 1;
+// Each cell's column, and the row-spanned cells that fill the slots before it
+// (or at the row's end): null for the rest of a cell's column span. A span
+// stops at its row group.
+function tableGrid(rows) {
+  const columns = new Map();
+  const before = new Map();
+  const after = new Map();
+  let carry = [];
+  let group = null;
+  for (const row of rows) {
+    if (row.parentNode !== group) carry = [];
+    group = row.parentNode;
+    let column = 0;
+    let slots = [];
+    const take = (gaps) => {
+      while (carry[column]?.left > 0 || (gaps && column < carry.length)) {
+        const entry = carry[column];
+        slots.push(entry?.left > 0 ? entry.cell : null);
+        if (entry?.left > 0) entry.left -= 1;
+        column += 1;
+      }
+    };
+    for (const cell of rowCells(row)) {
+      slots = [];
+      take();
+      before.set(cell, slots);
+      columns.set(cell, column);
+      const left = Number.parseInt(cell.getAttribute?.("rowspan") || "1", 10) - 1;
+      const span = spanOf(cell);
+      for (let i = 0; i < span; i += 1) {
+        if (left > 0) carry[column + i] = { cell: i ? null : cell, left };
+      }
+      column += span;
+    }
+    slots = [];
+    take(true);
+    after.set(row, slots);
+  }
+  return { columns, before, after };
 }
 
-// A GFM cell is one line, so a data table holding code blocks is rendered as
-// blocks too, each cell under its column's header: the code stays fenced.
-function isBlockTable(table) {
-  return isLayoutTable(table) || Boolean(table.querySelector?.("pre"));
+const tables = new WeakMap();
+
+// Worked out once per table: cell and row rules run for every cell.
+function tableInfo(table) {
+  let info = tables.get(table);
+  if (info) return info;
+  const rows = tableRows(table);
+  // Layout tables (nesting, a single cell, a single column) read better as
+  // blocks than as a grid.
+  const layout = Boolean(table.querySelector?.("table")) || !rows.length
+    || Math.max(...rows.map((row) => rowCells(row).reduce((sum, cell) => sum + spanOf(cell), 0))) <= 1;
+  // A GFM cell is one line, so a data table holding code blocks is rendered as
+  // blocks too, each cell under its column's header: the code stays fenced.
+  const block = layout || Boolean(table.querySelector?.("pre"));
+  info = { layout, block, header: columnHeaders(rows[0]), text: new Map(), ...tableGrid(rows) };
+  tables.set(table, info);
+  return info;
 }
 
 function isBlockCell(node) {
   const table = closest(node, "TABLE");
-  return !table || isBlockTable(table);
+  return !table || tableInfo(table).block;
+}
+
+// A row-spanned cell repeats in each row, so a row reads whole by itself.
+function slotsMarkdown(slots, info) {
+  return slots.map((cell) => ` ${info.text.get(cell) || ""} |`).join("");
 }
 
 function labelText(cell) {
@@ -92,8 +150,7 @@ function labelText(cell) {
 
 // A first row of two or more <th>: column headers. (A single spanning <th>
 // is a title, not a header.)
-function columnHeaders(table) {
-  const first = tableRows(table)[0];
+function columnHeaders(first) {
   const cells = first ? rowCells(first) : [];
   return cells.length > 1 && cells.every((cell) => cell.nodeName === "TH") ? first : null;
 }
@@ -108,16 +165,13 @@ function rowHeader(row) {
 // label; "" for layout tables and unlabelled cells.
 function cellLabel(cell) {
   const table = closest(cell, "TABLE");
-  if (!table || isLayoutTable(table)) return "";
+  const info = table && tableInfo(table);
+  if (!info || info.layout) return "";
   const row = cell.parentNode;
-  const header = columnHeaders(table);
+  const { header } = info;
   if (header) {
     if (row === header) return null;
-    let column = 0;
-    for (const other of rowCells(row)) {
-      if (other === cell) break;
-      column += spanOf(other);
-    }
+    const column = info.columns.get(cell);
     let start = 0;
     for (const th of rowCells(header)) {
       if (column < start + spanOf(th)) return labelText(th);
@@ -142,25 +196,83 @@ function cellText(content) {
   return text.replace(/\|/g, "\\|");
 }
 
+// An icon's straight strokes, as lists of points; null if any stroke curves.
+function iconStrokes(svg) {
+  const strokes = [];
+  for (const shape of Array.from(svg.querySelectorAll?.("path, polyline, line") || [])) {
+    const at = (name) => shape.getAttribute(name);
+    const d = at("d") ?? `M${at("points") ?? `${at("x1")} ${at("y1")} ${at("x2")} ${at("y2")}`}`;
+    const tokens = d.match(/[a-zA-Z]|-?\d*\.?\d+/g) || [];
+    let x = 0;
+    let y = 0;
+    let command = "";
+    let points = [];
+    for (let i = 0; i < tokens.length;) {
+      if (/[a-zA-Z]/.test(tokens[i])) command = tokens[i++];
+      const relative = command > "Z";
+      const type = command.toUpperCase();
+      if (!"MLHV".includes(type)) return null;
+      if (type !== "V") x = Number(tokens[i++]) + (relative ? x : 0);
+      if (type !== "H") y = Number(tokens[i++]) + (relative ? y : 0);
+      if (type === "M") {
+        points = [];
+        strokes.push(points);
+        command = relative ? "l" : "L";
+      }
+      points.push([x, y]);
+    }
+  }
+  return strokes;
+}
+
+// What an icon's shape says, for the tick, dash and cross of comparison tables.
+function iconShape(strokes) {
+  const [first, second] = strokes || [];
+  if (strokes?.length === 1 && first.length === 2 && first[0][1] === first[1][1]) return "\u2013";
+  if (strokes?.length === 1 && first.length === 3) {
+    const [a, b, c] = first;
+    const arm = (p) => Math.hypot(p[0] - b[0], p[1] - b[1]);
+    const ratio = Math.max(arm(a), arm(c)) / Math.min(arm(a), arm(c));
+    if (b[1] > a[1] && b[1] > c[1] && (a[0] - b[0]) * (c[0] - b[0]) < 0 && ratio > 1.3) return "\u2713";
+  }
+  const slope = ([p, q]) => (q[0] - p[0]) * (q[1] - p[1]);
+  if (strokes?.length === 2 && first.length === 2 && second.length === 2 && slope(first) * slope(second) < 0) return "\u2717";
+  return "";
+}
+
+// A cell holding only an icon would be empty, and the table couldn't say which
+// plan has what: its accessible name, else its shape, else that an icon is there.
+function cellIcon(cell) {
+  const svg = cell.querySelector?.("svg");
+  if (!svg) return "";
+  return svg.getAttribute("aria-label")?.trim() || svg.querySelector("title")?.textContent?.trim()
+    || iconShape(iconStrokes(svg)) || "[icon]";
+}
+
 function tableRules(turndown) {
   turndown.addRule("pmTableCell", {
     filter: ["th", "td"],
-    replacement(content, node) {
+    replacement(rawContent, node) {
+      const content = rawContent.trim() ? rawContent : cellIcon(node);
       if (isBlockCell(node)) {
         // Header cells become the labels on the other cells.
         const label = cellLabel(node);
         if (label === null) return "";
         return label ? `\n\n**${createTurndown().escape(label)}**\n\n${content}\n\n` : `\n\n${content}\n\n`;
       }
+      const info = tableInfo(closest(node, "TABLE"));
+      const text = cellText(content);
+      info.text.set(node, text);
       const pad = " |   ".repeat(spanOf(node) - 1);
-      return ` ${cellText(content)}${pad} |`;
+      return `${slotsMarkdown(info.before.get(node), info)} ${text}${pad} |`;
     },
   });
   turndown.addRule("pmTableRow", {
     filter: "tr",
     replacement(content, node) {
       if (isBlockCell(node)) return `\n${content}\n`;
-      return `\n|${content}`;
+      const info = tableInfo(closest(node, "TABLE"));
+      return `\n|${content}${slotsMarkdown(info.after.get(node), info)}`;
     },
   });
   turndown.addRule("pmTableSection", {
@@ -172,7 +284,7 @@ function tableRules(turndown) {
   turndown.addRule("pmTable", {
     filter: "table",
     replacement(content, node) {
-      if (isBlockTable(node)) return `\n\n${content}\n\n`;
+      if (tableInfo(node).block) return `\n\n${content}\n\n`;
       const lines = content.split("\n").filter((line) => line.startsWith("|"));
       if (!lines.length) return "\n\n";
       const width = Math.max(...lines.map((line) => line.split(/(?<!\\)\|/).length - 2));
@@ -197,7 +309,10 @@ function tableRules(turndown) {
 
 let shared = null;
 
-/** A configured Turndown instance (cached). */
+/**
+ * A configured Turndown instance (cached).
+ * @returns {import("./types.js").TurndownInstance}
+ */
 export function createTurndown() {
   if (shared) return shared;
   const turndown = new TurndownService({ ...TURNDOWN_OPTIONS });