@opencraw/core 0.1.3 → 0.1.4

package/dist/index.esm.js

@@ -7,6 +7,9 @@ import { createWriteStream } from 'node:fs';
 import { once } from 'node:events';
 import { JSONPath } from 'jsonpath-plus';
 import { load } from 'cheerio';
+import { gunzipSync } from 'node:zlib';
+import { DOMParser, DOMImplementation, XMLSerializer } from '@xmldom/xmldom';
+import { useNamespaces } from 'xpath';
 import { fileURLToPath } from 'node:url';
 
 /** Resource types a page may skip loading, to save bandwidth on per-GB proxies. */
@@ -2988,10 +2991,13 @@ const DOCUMENT = /^\s*(?:<!doctype|<html)/i;
  *
  * @param html - The markup (a whole document or a fragment).
  * @param selector - A CSS selector.
+ * @param xml - Read the markup as XML: names keep their case, `take: "html"` gives XML.
  * @returns Every match, in document order.
  */
-function selectHtml(html, selector) {
-  const api = DOCUMENT.test(html) ? load(html) : load(html, undefined, false);
+function selectHtml(html, selector, xml = false) {
+  const api = xml ? load(html, {
+    xml: true
+  }) : DOCUMENT.test(html) ? load(html) : load(html, undefined, false);
   return api(selector).map((_, element) => ({
     api,
     element: api(element)
@@ -4038,8 +4044,8 @@ function filledGrid(sheet) {
 /** `B10:B13` as 0-based bounds; `undefined` for anything else. */
 function rangeOf(reference) {
   const [from, to = from] = reference.split(':', 2);
-  const start = cellOf(from);
-  const end = cellOf(to);
+  const start = cellOf$1(from);
+  const end = cellOf$1(to);
   if (start === undefined || end === undefined) return undefined;
   return {
     top: Math.min(start.row, end.row),
@@ -4048,7 +4054,7 @@ function rangeOf(reference) {
     right: Math.max(start.column, end.column)
   };
 }
-function cellOf(reference) {
+function cellOf$1(reference) {
   const match = /^\$?([A-Z]+)\$?(\d+)$/i.exec(reference.trim());
   if (match === null) return undefined;
   const letters = match[1].toUpperCase();
@@ -4181,13 +4187,194 @@ function shapeTables(document, slides, query) {
   }));
 }
 
+/**
+ * Whether a value is a read XML document.
+ *
+ * @param value - Anything bound in a scope.
+ * @returns `true` for an `XmlDocument`.
+ */
+function isXmlDocument(value) {
+  return typeof value === 'object' && value !== null && value.kind === 'xml' && typeof value.xml === 'string';
+}
+
+/** The documents already parsed, by the text they came from, so a recipe's extracts parse a body once. */
+const parsed = new Map();
+const PARSED_LIMIT = 16;
+/**
+ * Parses XML strictly but safely: a malformed document is an error naming
+ * the URL and the line; entities a DOCTYPE declares are never expanded (no
+ * "billion laughs") and nothing external is ever fetched (no XXE), they stay
+ * as they are.
+ *
+ * @param xml - The document text.
+ * @param where - The URL or id, for messages.
+ * @returns The document.
+ * @throws Error when the text is not well-formed XML.
+ */
+function parseXml(xml, where) {
+  const cached = parsed.get(xml);
+  if (cached !== undefined) return cached;
+  let document;
+  try {
+    document = new DOMParser({
+      onError: failOnFatal
+    }).parseFromString(xml, 'text/xml');
+  } catch (error) {
+    throw new Error(`${where}: not well-formed XML (${firstLine(error)})`, {
+      cause: error
+    });
+  }
+  if (document.documentElement === null) throw new Error(`${where}: not an XML document (no root element)`);
+  remember(xml, document);
+  return document;
+}
+/**
+ * HTML as an XML document, for XPath on fetched pages: parsed the way a
+ * browser parses it (a forgiving HTML5 parser that inserts `<tbody>` and
+ * closes what the page left open), without namespaces, so `//table/tbody/tr`
+ * finds what it finds on the live page.
+ *
+ * @param html - A page or a fragment.
+ * @returns The document.
+ */
+function htmlAsXml(html) {
+  const cached = parsed.get(html);
+  if (cached !== undefined) return cached;
+  const whole = /^\s*(?:<!doctype|<html)/i.test(html);
+  // A fragment (a row taken with `take: "html"`) keeps its cells, and gets one root to be a document.
+  const xml = whole ? load(html).xml() : `<fragment>${load(html, undefined, false).xml()}</fragment>`;
+  const document = withoutNamespaces(new DOMParser({
+    onError: ignore
+  }).parseFromString(xml, 'text/xml'));
+  remember(html, document);
+  return document;
+}
+/**
+ * A copy of a document with every namespace dropped: elements and attributes
+ * keep their local names, `xmlns` declarations go. What `ignoreNamespaces`
+ * queries run on: `//entry/title` instead of `//atom:entry/atom:title`.
+ *
+ * @param document - A parsed document.
+ * @returns The copy.
+ */
+function withoutNamespaces(document) {
+  const copy = new DOMImplementation().createDocument(null, '', null);
+  const clone = node => {
+    if (node.nodeType === node.ELEMENT_NODE) {
+      const element = node;
+      const target = copy.createElement(element.localName);
+      for (const attribute of element.attributes) {
+        if (attribute.name === 'xmlns' || attribute.prefix === 'xmlns') continue;
+        target.setAttribute(attribute.localName, attribute.value);
+      }
+      for (const child of element.childNodes) {
+        const cloned = clone(child);
+        if (cloned !== undefined) target.insertBefore(cloned, null);
+      }
+      return target;
+    }
+    if (node.nodeType === node.TEXT_NODE || node.nodeType === node.CDATA_SECTION_NODE) return copy.createTextNode(node.nodeValue ?? '');
+    return undefined;
+  };
+  const root = document.documentElement === null ? undefined : clone(document.documentElement);
+  if (root !== undefined) copy.insertBefore(root, null);
+  return copy;
+}
+/**
+ * A node as markup: the element with its children (`outer`), or its children only.
+ *
+ * @param node - A node.
+ * @param outer - Whether to include the node itself.
+ * @returns The markup.
+ */
+function serializeXml(node, outer) {
+  const serializer = new XMLSerializer();
+  if (outer) return serializer.serializeToString(node);
+  return Array.from(node.childNodes, child => serializer.serializeToString(child)).join('');
+}
+/**
+ * The text of an XML body, gunzipped first when it is gzip (a `sitemap.xml.gz`).
+ *
+ * @param bytes - The body.
+ * @returns The bytes to decode.
+ */
+function gunzipIfNeeded(bytes) {
+  return bytes.length > 2 && bytes[0] === 0x1F && bytes[1] === 0x8B ? gunzipSync(bytes) : bytes;
+}
+function remember(text, document) {
+  if (parsed.size >= PARSED_LIMIT) parsed.delete(parsed.keys().next().value);
+  parsed.set(text, document);
+}
+function failOnFatal(level, message) {
+  if (level === 'fatalError') throw new Error(message);
+}
+function ignore() {}
+function firstLine(error) {
+  return (error instanceof Error ? error.message : String(error)).split('\n', 1)[0];
+}
+
+/**
+ * Runs an XPath 1.0 query on a parsed document. Prefixes the root element
+ * declares are known without being listed; a default namespace has no
+ * prefix, so name one in `namespaces` or use `ignoreNamespaces`.
+ *
+ * @param document - The document.
+ * @param expression - The query.
+ * @param options - Namespaces.
+ * @returns Every node selected, or the one value a function returned.
+ * @throws Error naming the query when it does not parse or uses an unknown prefix.
+ */
+function selectXpath(document, expression, options = {}) {
+  const target = options.ignoreNamespaces === true ? withoutNamespaces(document) : document;
+  const namespaces = {
+    ...declaredPrefixes(target),
+    ...options.namespaces
+  };
+  let result;
+  try {
+    result = useNamespaces(namespaces)(expression, target);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const hint = /prefix|namespace|QName/i.test(message) ? ' (declare it in "namespaces", or set "ignoreNamespaces": true)' : '';
+    throw new Error(`xpath ${expression}: ${message}${hint}`, {
+      cause: error
+    });
+  }
+  if (Array.isArray(result)) return result;
+  if (result === null) return [];
+  return [result];
+}
+/**
+ * The value of an XPath result.
+ *
+ * @param value - A node or a function result.
+ * @param take - `text` (whitespace collapsed), `html` (the node's inner markup), `json` (its outer markup), `value` (its text as is), `attr:<name>`.
+ * @returns The value; `undefined` for a missing attribute.
+ */
+function takeFromXml(value, take) {
+  if (typeof value !== 'object') return take === 'text' ? collapse(String(value)) : value;
+  if (take === 'text') return collapse(value.textContent ?? '');
+  if (take === 'value') return value.textContent ?? '';
+  if (take === 'html') return serializeXml(value, false);
+  if (take === 'json') return serializeXml(value, true);
+  if (value.nodeType !== value.ELEMENT_NODE) return undefined;
+  const element = value;
+  const name = take.slice('attr:'.length);
+  return element.hasAttribute(name) ? element.getAttribute(name) : undefined;
+}
+function declaredPrefixes(document) {
+  const root = document.documentElement;
+  if (root === null) return {};
+  return Object.fromEntries([...root.attributes].filter(attribute => attribute.prefix === 'xmlns').map(attribute => [attribute.localName, attribute.value]));
+}
+
 /**
  * Runs an `extract` step against a static document: the value bound under
- * `from`, else the scope's current document. `css` reads HTML, `jsonpath`
- * reads JSON (or a read PDF, workbook or deck as data), `table` reads the
- * tables of a PDF, a workbook (a spreadsheet, a CSV), a deck (a presentation)
- * or HTML (its `<table>`s), `regex` reads any document as text; `xpath` needs
- * a live page and is refused here.
+ * `from`, else the scope's current document. `css` reads HTML (or XML),
+ * `xpath` reads XML (a feed, a sitemap) or HTML parsed as a browser parses
+ * it, `jsonpath` reads JSON (or a read PDF, workbook or deck as data), `table`
+ * reads the tables of a PDF, a workbook (a spreadsheet, a CSV), a deck (a
+ * presentation) or HTML (its `<table>`s), `regex` reads any document as text.
  *
  * A `jsonpath` extract whose `from` is text parses that text as JSON, and a
  * list of texts (every `<script type="application/ld+json">` of a page) becomes
@@ -4206,6 +4393,7 @@ function extractFromDocument(step, scope) {
   switch (step.kind) {
     case 'jsonpath':
       {
+        if (document.kind === 'xml') throw new Error('jsonpath needs a JSON document; the current document is xml (read it with kind "xpath")');
         if (document.kind === 'html' || document.kind === 'text') throw new Error(`jsonpath needs a JSON document; the current document is ${document.kind}`);
         values = selectJson(document.kind === 'json' ? document.data : document, selector).map(node => takeFromJson(node, take));
         break;
@@ -4217,10 +4405,22 @@ function extractFromDocument(step, scope) {
       }
     case 'css':
       {
+        if (document.kind === 'xml') {
+          values = selectHtml(document.xml, selector, true).map(match => takeFromHtml(match, take));
+          break;
+        }
         if (document.kind !== 'html') throw new Error(`css needs an HTML document; the current document is ${document.kind}${['workbook', 'pdf', 'deck'].includes(document.kind) ? ' (read it with kind "table", "regex" or "jsonpath")' : ''}`);
         values = selectHtml(document.html, selector).map(match => takeFromHtml(match, take));
         break;
       }
+    case 'xpath':
+      {
+        values = selectXpath(xpathDocument(document), selector, {
+          namespaces: step.namespaces,
+          ignoreNamespaces: step.ignoreNamespaces
+        }).map(node => takeFromXml(node, take));
+        break;
+      }
     case 'regex':
       {
         values = selectRegex(textOf$1(document), selector);
@@ -4228,7 +4428,7 @@ function extractFromDocument(step, scope) {
       }
     default:
       {
-        throw new Error('xpath works on a live page only; use css on fetched HTML');
+        throw new Error(`unknown extract kind ${String(step.kind)}`);
       }
   }
   if (step.many === true) {
@@ -4334,9 +4534,32 @@ function patternOf(source, where) {
     });
   }
 }
+/** What an xpath extract queries: XML as it is, HTML parsed the way a browser would. */
+function xpathDocument(document) {
+  if (document.kind === 'xml') return parseXml(document.xml, 'the document');
+  if (document.kind === 'html') return htmlAsXml(document.html);
+  if (document.kind === 'text') return parseXml(document.text, 'the document');
+  throw new Error(`xpath reads XML or HTML; the current document is ${document.kind} (read it with kind ${document.kind === 'json' ? '"jsonpath"' : '"table", "regex" or "jsonpath"'})`);
+}
+/** Markup bound to an id: XML when it is well-formed (a feed entry taken with `take: "json"`), else HTML. */
+function markupDocument(markup) {
+  try {
+    parseXml(markup, 'markup');
+    return {
+      kind: 'xml',
+      xml: markup
+    };
+  } catch {
+    return {
+      kind: 'html',
+      html: markup
+    };
+  }
+}
 /** The text a regex extract reads: markup, text, a PDF's or a workbook's rows, or JSON re-serialised (a list of texts joined by newlines). */
 function textOf$1(document) {
   if (document.kind === 'html') return document.html;
+  if (document.kind === 'xml') return document.xml;
   if (document.kind === 'text') return document.text;
   if (document.kind === 'pdf') return pdfText(document);
   if (document.kind === 'workbook') return workbookText(document);
@@ -4352,7 +4575,11 @@ function documentFor(step, scope) {
   }
   const source = scope.get(step.from);
   if (source === undefined) throw new Error(`"${step.from}" is not bound`);
-  if (isPdfDocument(source) || isWorkbookDocument(source) || isDeckDocument(source)) return source;
+  if (isPdfDocument(source) || isWorkbookDocument(source) || isDeckDocument(source) || isXmlDocument(source)) return source;
+  if (step.kind === 'xpath') {
+    if (typeof source !== 'string') throw new Error(`"${step.from}" is not markup; xpath reads XML or HTML text`);
+    return markupDocument(source);
+  }
   if (step.kind === 'table') throw new Error(`"${step.from}" is not a PDF, a workbook or a deck; request it with "as": "pdf", "csv", "xlsx" or "pptx"`);
   if (step.kind === 'regex') {
     if (typeof source === 'string') return {
@@ -4441,6 +4668,12 @@ async function readYaml(text, source, scalars = 'typed') {
   };
 }
 
+/**
+ * `<` as the JSON escape `\u003c`, so a front matter value holding `</script>`
+ * cannot close its element. Built from char codes: a formatter would turn a
+ * literal escape back into `<`.
+ */
+const ESCAPED_LESS_THAN = String.fromCodePoint(92, 117, 48, 48, 51, 99);
 /** A leading `---` block of YAML. */
 const FRONT_MATTER = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/;
 /**
@@ -4474,14 +4707,22 @@ async function readMarkdown(text, source) {
     async: false
   });
   const sections = sectioned(rendered);
-  const head = front === undefined ? '' : `<script type="application/json" data-front-matter>${JSON.stringify(front.data ?? null).replaceAll('<', '<')}</script>`;
+  const head = front === undefined ? '' : `<script type="application/json" data-front-matter>${JSON.stringify(front.data ?? null).replaceAll('<', () => ESCAPED_LESS_THAN)}</script>`;
   return {
     html: `<!doctype html><html><head>${head}</head><body>${sections}</body></html>`,
     frontMatter: front?.data,
     warnings: front?.warnings ?? []
   };
 }
-/** Wraps each heading and what follows it, up to the next heading of the same or a higher level, in a section. */
+/**
+ * Wraps each heading and what follows it, up to the next heading of the same
+ * or a higher level, in `<section data-heading="…" data-level="…">`, sections
+ * nesting; headings get slug ids. Rendered Markdown and Word documents both go
+ * through it, so one selector finds "the table under *Prezzi*" in either.
+ *
+ * @param html - Top-level HTML: headings among paragraphs, lists, tables.
+ * @returns The same content, sectioned.
+ */
 function sectioned(html) {
   const $ = load(html, null, false);
   const open = [];
@@ -4502,7 +4743,7 @@ function sectioned(html) {
     const heading = $(node);
     const text = heading.text().replaceAll(/\s+/g, ' ').trim();
     heading.attr('id', uniqueSlug(text, used));
-    out += `<section data-heading="${escapeAttribute(text)}" data-level="${depth}">${$.html(node)}`;
+    out += `<section data-heading="${escapeAttribute$1(text)}" data-level="${depth}">${$.html(node)}`;
     open.push(depth);
   }
   return out + '</section>'.repeat(open.length);
@@ -4514,10 +4755,165 @@ function uniqueSlug(text, used) {
   used.set(slug, count + 1);
   return count === 0 ? slug : `${slug}-${count}`;
 }
-function escapeAttribute(text) {
+function escapeAttribute$1(text) {
   return text.replaceAll('&', '&amp;').replaceAll('"', '&quot;').replaceAll('<', '&lt;');
 }
 
+/**
+ * Reads a `.docx` Word document into HTML, through `@opencraw/office-reader`,
+ * so every selector that reads a page reads a Word document too:
+ *
+ * - headings become `<h1>`…`<h6>` inside `<section data-heading="…">`, as in
+ *   rendered Markdown;
+ * - list items become nested `<ul>` / `<ol>`;
+ * - tables become `<table>`s with their merged cells as `colspan` / `rowspan`,
+ *   so a `table` extract reads them like any HTML table;
+ * - a paragraph keeps its style as `data-style` and its links as `<a href>`;
+ * - headers, footers and notes follow the body, in `<header>`, `<footer>`
+ *   and `<aside data-part="notes">`; the title goes to `<title>`.
+ *
+ * The reader is imported on first use.
+ *
+ * @param bytes - The file.
+ * @param source - Where it came from, for messages.
+ * @returns The HTML document.
+ * @throws Error naming the source, and saying what to do, for a file that is
+ * not a readable Word document (a legacy `.doc`, a password-protected file, an `.odt`…).
+ */
+async function readDocxHtml(bytes, source) {
+  const {
+    readDocx,
+    OfficeReadError
+  } = await import('@opencraw/office-reader/docx');
+  let document;
+  try {
+    document = await readDocx(bytes);
+  } catch (error) {
+    if (error instanceof OfficeReadError) throw new Error(`${source}: ${error.message}`, {
+      cause: error
+    });
+    throw error;
+  }
+  const head = document.title === undefined ? '' : `<title>${escapeText(document.title)}</title>`;
+  const headers = document.headers.map(blocks => `<header data-part="header">${blocksHtml(blocks)}</header>`).join('');
+  const footers = document.footers.map(blocks => `<footer data-part="footer">${blocksHtml(blocks)}</footer>`).join('');
+  const notes = document.notes.length === 0 ? '' : `<aside data-part="notes"><ol>${document.notes.map(note => `<li id="${note.kind}-${escapeAttribute(note.id)}" data-kind="${note.kind}">${textHtml(note.text)}</li>`).join('')}</ol></aside>`;
+  return `<!doctype html><html><head>${head}</head><body>${headers}${sectioned(blocksHtml(document.body))}${footers}${notes}</body></html>`;
+}
+/** Blocks as HTML: headings, paragraphs, nested lists, tables. */
+function blocksHtml(blocks) {
+  let html = '';
+  const lists = [];
+  const closeLists = depth => {
+    while (lists.length > depth) {
+      const list = lists.pop();
+      html += `${list?.itemOpen === true ? '</li>' : ''}</${list?.tag ?? 'ul'}>`;
+    }
+  };
+  for (const block of blocks) {
+    if (block.kind === 'table') {
+      closeLists(0);
+      html += tableHtml(block);
+      continue;
+    }
+    if (block.list === undefined) {
+      closeLists(0);
+      html += paragraphHtml(block);
+      continue;
+    }
+    const tag = block.list.ordered ? 'ol' : 'ul';
+    const depth = block.list.level + 1;
+    closeLists(depth);
+    if (lists.length === depth && lists.at(-1)?.tag !== tag) closeLists(depth - 1);
+    while (lists.length < depth) {
+      html += `<${tag}>`;
+      lists.push({
+        tag,
+        itemOpen: false
+      });
+    }
+    const list = lists[depth - 1];
+    html += `${list.itemOpen ? '</li>' : ''}<li${styleAttribute(block)}>${inlineHtml(block)}`;
+    list.itemOpen = true;
+  }
+  closeLists(0);
+  return html;
+}
+function paragraphHtml(paragraph) {
+  if (paragraph.heading !== undefined) {
+    const level = Math.min(paragraph.heading, 6);
+    return `<h${level}${styleAttribute(paragraph)}>${inlineHtml(paragraph)}</h${level}>`;
+  }
+  return `<p${styleAttribute(paragraph)}>${inlineHtml(paragraph)}</p>`;
+}
+/** A paragraph's text with its links as anchors, in the order they appear. */
+function inlineHtml(paragraph) {
+  let html = '';
+  let rest = paragraph.text;
+  const links = paragraph.links ?? [];
+  for (const link of links) {
+    const at = rest.indexOf(link.text);
+    if (at === -1) continue;
+    html += `${textHtml(rest.slice(0, at))}<a href="${escapeAttribute(link.href)}">${textHtml(link.text)}</a>`;
+    rest = rest.slice(at + link.text.length);
+  }
+  return html + textHtml(rest);
+}
+/** A grid as a table: the top-left cell of a merged range spans it, the cells it covers are left out. */
+function tableHtml(table) {
+  const spans = new Map();
+  const covered = new Set();
+  for (const range of table.merges) {
+    const [from, to] = range.split(':').map(reference => cellOf(reference));
+    if (from === undefined || to === undefined) continue;
+    spans.set(`${from.row},${from.column}`, {
+      rows: to.row - from.row + 1,
+      columns: to.column - from.column + 1
+    });
+    for (let row = from.row; row <= to.row; row += 1) {
+      for (let column = from.column; column <= to.column; column += 1) {
+        if (row !== from.row || column !== from.column) covered.add(`${row},${column}`);
+      }
+    }
+  }
+  const rows = table.rows.map((cells, row) => {
+    const html = cells.map((text, column) => {
+      const key = `${row},${column}`;
+      if (covered.has(key)) return '';
+      const span = spans.get(key);
+      const attributes = span === undefined ? '' : `${span.rows > 1 ? ` rowspan="${span.rows}"` : ''}${span.columns > 1 ? ` colspan="${span.columns}"` : ''}`;
+      return `<td${attributes}>${textHtml(text)}</td>`;
+    }).join('');
+    return `<tr>${html}</tr>`;
+  }).join('');
+  return `<table data-name="${escapeAttribute(table.name)}">${rows}</table>`;
+}
+/** An A1 reference as 0-based row and column. */
+function cellOf(reference) {
+  const match = /^([A-Z]+)(\d+)$/.exec(reference);
+  if (match === null) return undefined;
+  let column = 0;
+  const letters = match[1];
+  for (const letter of letters) column = column * 26 + (letter.codePointAt(0) ?? 64) - 64;
+  return {
+    row: Number(match[2]) - 1,
+    column: column - 1
+  };
+}
+function styleAttribute(paragraph) {
+  return paragraph.style === undefined ? '' : ` data-style="${escapeAttribute(paragraph.style)}"`;
+}
+/** Text as HTML: escaped, line breaks as `<br>`. */
+function textHtml(text) {
+  return escapeText(text).replaceAll('\n', '<br>');
+}
+function escapeText(text) {
+  return text.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;');
+}
+function escapeAttribute(text) {
+  return escapeText(text).replaceAll('"', '&quot;');
+}
+
 /** A response with a 4xx or 5xx status. */
 class HttpError extends Error {
   status;
@@ -4677,7 +5073,7 @@ class HttpClient {
 }
 async function readBody(response, httpRequest) {
   const contentType = response.headers()['content-type'] ?? '';
-  const format = httpRequest.as ?? formatFromContentType(contentType);
+  const format = httpRequest.as ?? formatFromContentType(contentType, response.url());
   return parseBody(format, await response.body(), response.url(), {
     ...httpRequest,
     charset: charsetOf(contentType)
@@ -4695,7 +5091,7 @@ async function readLocalFile(httpRequest) {
     body,
     warnings,
     format
-  } = await parseBody(httpRequest.as ?? formatFromExtension(extname(path)), bytes, httpRequest.url, httpRequest);
+  } = await parseBody(httpRequest.as ?? formatFromExtension(path), bytes, httpRequest.url, httpRequest);
   return {
     status: 200,
     url: httpRequest.url,
@@ -4725,6 +5121,35 @@ async function parseBody(format, bytes, url, reading) {
       format
     };
   }
+  if (format === 'xml') {
+    // A sitemap is often served gzipped as a file (`sitemap.xml.gz`), not as a compressed response.
+    const {
+      text
+    } = decodeText(gunzipIfNeeded(bytes), reading);
+    try {
+      parseXml(text, url);
+    } catch (error) {
+      throw new Error(`${error.message}${/^\s*(?:<!doctype html|<html)/i.test(text) ? '; the body is HTML: read it with "as": "html"' : ''}`, {
+        cause: error
+      });
+    }
+    return {
+      body: {
+        kind: 'xml',
+        xml: text
+      },
+      warnings: [],
+      format
+    };
+  }
+  if (format === 'docx') return {
+    body: {
+      kind: 'html',
+      html: await readDocxHtml(bytes, url)
+    },
+    warnings: [],
+    format
+  };
   if (format === 'markdown') {
     const {
       text
@@ -4790,24 +5215,31 @@ function looksLikeJsonLines(text) {
   const first = parseJsonLike(lines[0]);
   return 'value' in first;
 }
-function formatFromContentType(contentType) {
+function formatFromContentType(contentType, url) {
   const type = contentType.toLowerCase().split(';', 1)[0].trim();
+  // A gzipped sitemap is served as an archive; its name says what is inside.
+  if (GZIP_TYPES.has(type) && /\.xml\.gz$/i.test(new URL(url).pathname)) return 'xml';
   if (CSV_TYPES.has(type)) return 'csv';
   if (JSON_LINES_TYPES.has(type)) return 'jsonl';
   // A legacy .xls or .ppt goes to the Office reader too, which says what to do with it.
   if (type.includes('spreadsheetml') || type.startsWith('application/vnd.ms-excel')) return 'xlsx';
   if (type.includes('presentationml') || type.startsWith('application/vnd.ms-powerpoint')) return 'pptx';
+  if (type === 'application/msword' || type.includes('wordprocessingml') || type.startsWith('application/vnd.ms-word')) return 'docx';
   if (YAML_TYPES.has(type)) return 'yaml';
   if (type === 'text/markdown' || type === 'text/x-markdown') return 'markdown';
   if (type.includes('json')) return 'json';
   if (type.includes('pdf')) return 'pdf';
-  if (type.includes('html') || type.includes('xml')) return 'html';
+  if (type.includes('html')) return 'html';
+  if (type.includes('xml')) return 'xml';
   return 'text';
 }
 const JSON_LINES_TYPES = new Set(['application/x-ndjson', 'application/ndjson', 'application/jsonl', 'application/x-jsonlines', 'application/jsonlines']);
 const YAML_TYPES = new Set(['application/yaml', 'application/x-yaml', 'text/yaml', 'text/x-yaml']);
+const GZIP_TYPES = new Set(['application/gzip', 'application/x-gzip', 'application/octet-stream']);
 const CSV_TYPES = new Set(['text/csv', 'application/csv', 'text/x-csv', 'application/x-csv', 'text/comma-separated-values', 'text/tab-separated-values']);
-function formatFromExtension(extension) {
+function formatFromExtension(path) {
+  if (/\.xml\.gz$/i.test(path)) return 'xml';
+  const extension = extname(path);
   const formats = {
     '.json': 'json',
     '.jsonl': 'jsonl',
@@ -4822,13 +5254,21 @@ function formatFromExtension(extension) {
     '.pptm': 'pptx',
     '.ppsx': 'pptx',
     '.ppt': 'pptx',
+    '.docx': 'docx',
+    '.docm': 'docx',
+    '.dotx': 'docx',
+    '.doc': 'docx',
     '.yaml': 'yaml',
     '.yml': 'yaml',
     '.md': 'markdown',
     '.markdown': 'markdown',
     '.html': 'html',
     '.htm': 'html',
-    '.xml': 'html'
+    '.xml': 'xml',
+    '.rss': 'xml',
+    '.atom': 'xml',
+    '.kml': 'xml',
+    '.gpx': 'xml'
   };
   return formats[extension.toLowerCase()] ?? 'text';
 }
@@ -4937,6 +5377,7 @@ function bodyText(body) {
   if (body.kind === 'pdf') return pdfText(body);
   if (body.kind === 'workbook') return workbookText(body);
   if (body.kind === 'deck') return deckText(body);
+  if (body.kind === 'xml') return body.xml;
   return body.kind === 'html' ? body.html : body.text;
 }
 function renderMap(map, lookup) {
@@ -7189,7 +7630,7 @@ const CRAWL_MODES = ['web', 'api'];
 const SELECTOR_KINDS = ['css', 'xpath', 'jsonpath', 'regex', 'table'];
 /** `take` also accepts `attr:<name>`, which is validated by pattern rather than listed. */
 const TAKE_KINDS = ['text', 'html', 'value', 'json'];
-const BODY_KINDS = ['json', 'jsonl', 'html', 'text', 'pdf', 'csv', 'xlsx', 'pptx', 'yaml', 'markdown'];
+const BODY_KINDS = ['json', 'jsonl', 'html', 'text', 'pdf', 'csv', 'xlsx', 'pptx', 'yaml', 'markdown', 'xml', 'docx'];
 const YAML_SCALARS = ['typed', 'text'];
 /** How a PDF table aligns a row's values against a cell wrapped over several lines. */
 const TABLE_ALIGNS = ['auto', 'top', 'center', 'bottom'];
@@ -7395,8 +7836,20 @@ const extractStep = z.strictObject({
   fillDown: z.array(z.string().min(1)).min(1).optional(),
   includeHidden: z.boolean().optional(),
   slide: z.string().min(1).optional(),
-  shapes: z.boolean().optional()
+  shapes: z.boolean().optional(),
+  namespaces: z.record(z.string().regex(/^[A-Z_][\w.-]*$/i, 'a namespace prefix such as atom'), z.string().min(1)).optional(),
+  ignoreNamespaces: z.boolean().optional()
 }).check(context => {
+  if (context.value.kind !== 'xpath') {
+    for (const key of ['namespaces', 'ignoreNamespaces']) {
+      if (context.value[key] !== undefined) context.issues.push({
+        code: 'custom',
+        input: context.value,
+        path: [key],
+        message: `"${key}" belongs to kind "xpath"`
+      });
+    }
+  }
   if (context.value.kind === 'table') return;
   for (const key of tableOnly) {
     if (context.value[key] !== undefined) context.issues.push({
@@ -8190,5 +8643,176 @@ function isSameOutput(document, output) {
   return recipeKindOf(document.content) === 'output' && document.content.id === output.id;
 }
 
-export { ACCESS_PRESETS, AccessBroker, AccessConfigError, BrowserClient, BrowserSession, CaptchaError, DEFAULT_CAPTCHA_SELECTOR, DEFAULT_RETRY_RULE, HostThrottle, HttpClient, HttpError, MappingFailedError, PdfReadError, RecipeBindingError, RecipeSet, RecipeValidationError, RecordRejectedError, StepFailure, TransformError, UnknownHookError, accessConfigJsonSchema, accessConfigSchema, bindRecipeSet, createCrawler, csvWorkbook, deckText, detectChallenge, detectDelimiter, fillDown, findDeckTables, findGridTables, findTables, htmlTableSheets, inputRecipeJsonSchema, inputRecipeSchema, isDeckDocument, isWorkbookDocument, jsonLinesSink, loadAccessConfig, loadRecipeSet, loadRecipes, memorySink, outputRecipeJsonSchema, outputRecipeSchema, parseCsv, parseInputRecipe, parseOutputRecipe, pdfText, readMarkdown, readPdf, readRecipeSource, readYaml, retryRuleSchema, throttleConfigSchema, traceLine, tryParseJson, validateBinding, workbookText };
+const BOOKKEEPING = new Set(['_source', '_key']);
+/**
+ * Compares two runs' records by key: what was added, what was removed, and
+ * for the records in both, which fields changed, before and after. Like a
+ * `git diff` of a price list, but row by row instead of line by line, so a
+ * reordered file is not a change.
+ *
+ * @param previous - The earlier run's records.
+ * @param current - The later run's records.
+ * @param options - The key fields, the fields to ignore, the shrink threshold.
+ * @returns The differences.
+ * @throws Error when records carry no key: without `key`, every line needs `_key`.
+ */
+function diffRecords(previous, current, options = {}) {
+  const ignore = new Set([...BOOKKEEPING, ...(options.ignore ?? [])]);
+  const keyOf = keyReader(options.key);
+  let repeated = 0;
+  const index = (records, side) => {
+    const byKey = new Map();
+    for (const [line, record] of records.entries()) {
+      const key = keyOf(record, `${side} record ${line + 1}`);
+      if (byKey.has(key)) repeated += 1;else byKey.set(key, record);
+    }
+    return byKey;
+  };
+  const before = index(previous, 'previous');
+  const after = index(current, 'current');
+  const removed = [];
+  const changed = [];
+  const added = [];
+  let unchanged = 0;
+  for (const [key, record] of before) {
+    const now = after.get(key);
+    if (now === undefined) {
+      removed.push({
+        change: 'removed',
+        key,
+        before: record
+      });
+      continue;
+    }
+    const fields = fieldChanges(record, now, ignore, '');
+    if (fields.length === 0) unchanged += 1;else changed.push({
+      change: 'changed',
+      key,
+      before: record,
+      after: now,
+      fields
+    });
+  }
+  for (const [key, record] of after) {
+    if (!before.has(key)) added.push({
+      change: 'added',
+      key,
+      after: record
+    });
+  }
+  const threshold = options.shrink ?? 0.5;
+  const shrunk = previous.length > 0 && current.length < previous.length * (1 - threshold);
+  return {
+    added: added.length,
+    removed: removed.length,
+    changed: changed.length,
+    unchanged,
+    changes: [...removed, ...changed, ...added],
+    repeated,
+    counts: {
+      previous: previous.length,
+      current: current.length
+    },
+    ...(shrunk && {
+      shrunk: {
+        previous: previous.length,
+        current: current.length
+      }
+    })
+  };
+}
+/**
+ * The diff options an output recipe implies: its key fields, and its fields
+ * the engine fills differently every run (`generated: now`, `uuid`).
+ *
+ * @param output - The output recipe.
+ * @returns The key and the fields to ignore.
+ */
+function diffOptionsFor(output) {
+  const fields = Object.entries(output.fields);
+  return {
+    key: fields.filter(([, field]) => field.key === true).map(([name]) => name),
+    ignore: fields.filter(([, field]) => field.generated === 'now' || field.generated === 'uuid').map(([name]) => name)
+  };
+}
+/**
+ * A key as people read it: the key fields' values joined (`Pandina · 1.0 Hybrid`).
+ *
+ * @param key - A record key (a JSON array of the key values).
+ * @returns The readable form.
+ */
+function readableKey(key) {
+  try {
+    const values = JSON.parse(key);
+    return Array.isArray(values) ? values.map(value => typeof value === 'string' ? value : JSON.stringify(value)).join(' · ') : key;
+  } catch {
+    return key;
+  }
+}
+function keyReader(fields) {
+  if (fields !== undefined && fields.length > 0) return record => JSON.stringify(fields.map(field => record[field] ?? null));
+  return storedKey;
+}
+/** The `_key` a sink in append mode wrote on the line. */
+function storedKey(record, where) {
+  const key = record._key;
+  if (typeof key !== 'string') throw new Error(`${where} has no _key: name the fields that identify a record (--key model,version, or the output recipe's key fields)`);
+  return key;
+}
+function fieldChanges(before, after, ignore, prefix) {
+  const names = [...new Set([...Object.keys(before), ...Object.keys(after)])];
+  const changes = [];
+  for (const name of names) {
+    const field = `${prefix}${name}`;
+    if (ignore.has(field) || prefix === '' && ignore.has(name)) continue;
+    const was = before[name];
+    const is = after[name];
+    if (isPlainObject(was) && isPlainObject(is)) changes.push(...fieldChanges(was, is, ignore, `${field}.`));else if (!sameValue(was, is)) changes.push({
+      field,
+      before: was ?? null,
+      after: is ?? null
+    });
+  }
+  return changes;
+}
+function isPlainObject(value) {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/** Equal as JSON, whatever the order of object keys. */
+function sameValue(first, second) {
+  return stableJson(first ?? null) === stableJson(second ?? null);
+}
+function stableJson(value) {
+  if (Array.isArray(value)) return `[${value.map(entry => stableJson(entry)).join(',')}]`;
+  if (isPlainObject(value)) return `{${Object.keys(value).sort((first, second) => first.localeCompare(second)).map(key => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(',')}}`;
+  return JSON.stringify(value) ?? 'null';
+}
+
+/**
+ * The records of a JSON Lines file, as a sink wrote them.
+ *
+ * @param path - The file.
+ * @returns One record per non-empty line.
+ * @throws Error naming the file and the line that is not a JSON object.
+ */
+async function readRecordsFile(path) {
+  const text = await readFile(path, 'utf8');
+  const records = [];
+  for (const [index, line] of text.split(/\r?\n/).entries()) {
+    if (line.trim() === '') continue;
+    let value;
+    try {
+      value = JSON.parse(line);
+    } catch (error) {
+      throw new Error(`${path}:${index + 1}: not JSON (${error.message})`, {
+        cause: error
+      });
+    }
+    if (typeof value !== 'object' || value === null || Array.isArray(value)) throw new Error(`${path}:${index + 1}: not a record (a JSON object)`);
+    records.push(value);
+  }
+  return records;
+}
+
+export { ACCESS_PRESETS, AccessBroker, AccessConfigError, BrowserClient, BrowserSession, CaptchaError, DEFAULT_CAPTCHA_SELECTOR, DEFAULT_RETRY_RULE, HostThrottle, HttpClient, HttpError, MappingFailedError, PdfReadError, RecipeBindingError, RecipeSet, RecipeValidationError, RecordRejectedError, StepFailure, TransformError, UnknownHookError, accessConfigJsonSchema, accessConfigSchema, bindRecipeSet, createCrawler, csvWorkbook, deckText, detectChallenge, detectDelimiter, diffOptionsFor, diffRecords, fillDown, findDeckTables, findGridTables, findTables, htmlTableSheets, inputRecipeJsonSchema, inputRecipeSchema, isDeckDocument, isWorkbookDocument, isXmlDocument, jsonLinesSink, loadAccessConfig, loadRecipeSet, loadRecipes, memorySink, outputRecipeJsonSchema, outputRecipeSchema, parseCsv, parseInputRecipe, parseOutputRecipe, parseXml, pdfText, readMarkdown, readPdf, readRecipeSource, readRecordsFile, readYaml, readableKey, retryRuleSchema, selectXpath, takeFromXml, throttleConfigSchema, traceLine, tryParseJson, validateBinding, workbookText };
 //# sourceMappingURL=index.esm.js.map