@bettercms-ai/richtext 0.12.1 → 0.16.0

package/src/ingest.ts

@@ -3,6 +3,7 @@ import { compileSchema } from "@portabletext/schema";
 import {
   BUILTIN_EMBED_KEY,
   BUILTIN_IMAGE_KEY,
+  BUILTIN_SITE_KEY,
   PT_TYPES,
   type PortableText,
   type PortableTextBlock,
@@ -15,8 +16,13 @@ import {
 import { randomKey, isValidKey, type KeyGenerator } from "./key";
 import { safeHref } from "./href";
 import { BARE_IS_DESIGN, isStyledTag, styledAttrsOf, type StyledMarkDef } from "./styled";
+import { EMBED_TITLE_MAX, isEmbedAspect, nearestEmbedAspect, resolveEmbed } from "./embed";
 import { cellTypeOf, columnOf, designOf, isColumnType, isValueType, jsonLdOf, PT_TABLE_LIMITS, typedCellText } from "./table";
 import { PT_KEY_TAKEAWAYS_LIMITS } from "./key-takeaways";
+import { PT_GALLERY_LIMITS } from "./gallery";
+import { PT_TOC_LIMITS, isHeadingAnchor, outlineOf } from "./toc";
+import { PT_RELATED_LIMITS } from "./references";
+import { readImageFrame } from "./image-layout";
 import type { PtSchema } from "./schema";
 
 /**
@@ -48,6 +54,25 @@ export interface IngestOptions {
   /** What the field accepts. Anything outside it produces no node. */
   schema?: PtSchema;
   keyGenerator?: KeyGenerator;
+  /**
+   * Keep parts the site renders (`builtin:site` markers) where the schema has no generic block.
+   * Only the marker is admitted: the schema still decides every other node.
+   */
+  siteParts?: boolean;
+  /**
+   * Called once per fact an import receipt records, today `embed.legacy_iframe:<host>`: an
+   * `<iframe>` the embed allowlist does not frame, kept on the legacy read-only path. The evidence
+   * the allowlist grows from.
+   */
+  onReceipt?: (line: string) => void;
+  /**
+   * RT9: keep a top-level h2/h3's own `id` as the block's `anchor` (only when it differs from the
+   * id its words make). OFF by default, because a value stored before anchors existed was read
+   * without them: a comparison of such a value with its page must read the page the same way, or
+   * an unedited page reads as changed (R15). An import turns it on; a comparison turns it on only
+   * when the stored value has an anchor (`hasHeadingAnchors`).
+   */
+  headingAnchors?: boolean;
 }
 
 /**
@@ -460,6 +485,29 @@ const codeLanguage = (pre: Element): string | undefined =>
  * deserialiser's default flattens a code block to a paragraph with a `code` mark and its
  * newlines collapsed: every imported code fence became one line of inline code.
  */
+/**
+ * The marker a part the SITE renders is written as (`<div class="bcms-block" data-bcms-schema-key=
+ * "builtin:site">`, @see BUILTIN_SITE_KEY) → that locked block, never its contents as paragraphs.
+ * The repo source and the card's label ride on `data-bcms-site-source` / `data-bcms-site-label`,
+ * which only the source lane writes; a rendered marker carries neither and still comes back locked.
+ */
+function siteRule(key: KeyGenerator): DeserializerRule {
+  return {
+    deserialize(el: Node, _next, createBlock) {
+      if (!isElement(el) || el.nodeName.toLowerCase() !== "div" || el.getAttribute("data-bcms-schema-key") !== BUILTIN_SITE_KEY) return undefined;
+      const source = el.getAttribute("data-bcms-site-source") ?? "";
+      const label = el.getAttribute("data-bcms-site-label") ?? "";
+      return createBlock({
+        _type: PT_TYPES.block,
+        _key: key(),
+        schemaKey: BUILTIN_SITE_KEY,
+        schemaLabel: label || "Site component",
+        fields: source ? { source } : {},
+      });
+    },
+  };
+}
+
 function codeRule(key: KeyGenerator): DeserializerRule {
   return {
     deserialize(el: Node, _next, createBlock) {
@@ -513,6 +561,216 @@ function styledRule(key: KeyGenerator): DeserializerRule {
   };
 }
 
+/**
+ * The two decorators the renderer writes in markup the standard rules do not read back (R1):
+ * - `<mark>`: the deserialiser names it `mark`, the schema calls it `highlight`, so it was
+ *   dropped on every round trip;
+ * - `<span style="text-decoration:underline">`: what `portableTextToHtml` emits for `underline`,
+ *   which the styled rule then recorded as the site's own design instead of the decorator.
+ * Bare only: a classed `<mark>` or a span with anything else on it is still the styled rule's.
+ * Gated on the schema, because a custom rule's output is never filtered by it.
+ */
+function ownDecoratorRule(schema: PtSchema | undefined): DeserializerRule {
+  const allows = (name: string) => !schema || schema.decorators.some((d) => d.name === name);
+  return {
+    deserialize(el: Node, next) {
+      if (!isElement(el)) return undefined;
+      const tag = el.nodeName.toLowerCase();
+      const attrs = el.getAttributeNames();
+      if (tag === "mark" && attrs.length === 0 && allows("highlight")) {
+        return { _type: "__decorator", name: "highlight", children: next(el.childNodes) } as never;
+      }
+      const style = el.getAttribute("style")?.replace(/\s+/g, "").replace(/;$/, "").toLowerCase();
+      if (tag === "span" && attrs.length === 1 && style === "text-decoration:underline" && allows("underline")) {
+        return { _type: "__decorator", name: "underline", children: next(el.childNodes) } as never;
+      }
+      return undefined;
+    },
+  };
+}
+
+/**
+ * A V2 checklist item, `<ul data-task="true"><li data-checked="true|false">`, read as
+ * `listItem: "check"` + `checked`; a nested list's depth is added when it is flattened. The box in front of the words (the
+ * renderer's disabled checkbox) is skipped: the state is `data-checked`, never the input. Gated on
+ * the schema, because a custom rule's output is never filtered by it: a field without the
+ * checklist falls through to the standard rule and reads the items as bullets.
+ */
+function checklistItemRule(schema: PtSchema | undefined): DeserializerRule {
+  const allowed = !schema || schema.lists.some((l) => l.name === "check");
+  return {
+    deserialize(el: Node, next) {
+      if (!allowed || !isElement(el) || el.nodeName.toLowerCase() !== "li") return undefined;
+      const list = el.parentNode;
+      if (!list || !isElement(list) || list.nodeName.toLowerCase() !== "ul" || !list.hasAttribute("data-task")) return undefined;
+      const words = Array.from(el.childNodes).filter((n) => !(isElement(n) && n.nodeName.toLowerCase() === "input"));
+      return {
+        _type: "block",
+        style: "normal",
+        markDefs: [],
+        listItem: "check",
+        // 1, as the standard list rule writes it: flattening a nested list adds the depth.
+        level: 1,
+        checked: el.getAttribute("data-checked") === "true",
+        children: next(words as never),
+      } as never;
+    },
+  };
+}
+
+/**
+ * RT8: `<aside data-rt="related">` → `relatedPosts` and `<aside data-rt="author">` → `authorBio`, the
+ * inverses of their renderers and of V2's markup. Only what is stored is read: the title (≤ 40),
+ * the mode (V2's "auto" is "topic"; else "newest" or "manual"), the count (V2 clamps it to 1–6,
+ * default 3) and the picks (`data-posts`); the author's id (`data-author`). The listed posts and
+ * the author's name, avatar and bio are resolved at render and ignored.
+ */
+function referenceRule(key: KeyGenerator, schema: PtSchema | undefined): DeserializerRule {
+  const related = allowsBlockObject(schema, PT_TYPES.relatedPosts);
+  const author = allowsBlockObject(schema, PT_TYPES.authorBio);
+  return {
+    deserialize(el: Node, _next, createBlock) {
+      if (!isElement(el) || el.nodeName.toLowerCase() !== "aside") return undefined;
+      const rt = el.getAttribute("data-rt");
+      if (rt === "related" && related) {
+        // Our placeholder (no `resolved`) carries the title as an attribute; a filled aside, and V2's, as `[data-title]`.
+        const title = (el.getAttribute("data-title") ?? el.querySelector("[data-title]")?.textContent ?? "").replace(/\s+/g, " ").trim().slice(0, PT_RELATED_LIMITS.titleChars);
+        const m = el.getAttribute("data-mode");
+        const mode = m === "manual" || m === "newest" ? m : "topic";
+        const count = Math.min(PT_RELATED_LIMITS.maxCount, Math.max(PT_RELATED_LIMITS.minCount, Math.trunc(Number(el.getAttribute("data-count"))) || 3));
+        const posts = (el.getAttribute("data-posts") ?? "").split(",").map((id) => id.trim()).filter(Boolean).map((_ref) => ({ _key: key(), _ref }));
+        return createBlock({ _type: PT_TYPES.relatedPosts, _key: key(), ...(title ? { title } : {}), mode, count, posts });
+      }
+      if (rt === "author" && author) {
+        const id = (el.getAttribute("data-author") ?? "").trim();
+        return createBlock({ _type: PT_TYPES.authorBio, _key: key(), ...(id ? { author: { _ref: id } } : {}) });
+      }
+      return undefined;
+    },
+  };
+}
+
+/**
+ * RT7: `<div data-rt="columns" data-gap>` → the `columns` block, the inverse of `columnsHtml` and
+ * of V2's own markup: one column per `[data-col]` child, its html read with the same options as
+ * the body (the field's schema too), and the gap (s, m, l; else m). The count is derived. Nothing
+ * is dropped here: a column holding more than text, or a count outside 2 to 4, is the write check's
+ * to refuse by name.
+ */
+function columnsRule(key: KeyGenerator, inner: (html: string) => PortableText): DeserializerRule {
+  return {
+    deserialize(el: Node, _next, createBlock) {
+      if (!isElement(el) || el.nodeName.toLowerCase() !== "div" || el.getAttribute("data-rt") !== "columns") return undefined;
+      const g = el.getAttribute("data-gap");
+      const columns = Array.from(el.children)
+        .filter((c) => c.hasAttribute("data-col"))
+        .map((c) => ({ _key: key(), content: inner(c.innerHTML) }));
+      return createBlock({ _type: PT_TYPES.columns, _key: key(), gap: g === "s" || g === "l" ? g : "m", columns });
+    },
+  };
+}
+
+/**
+ * RT5: `<nav data-rt="toc" data-depth>` → the `tableOfContents` block, the inverse of `tocHtml`
+ * and of V2's own markup. Only what is stored is read: the title (`[data-title]`, ≤ 40, none when
+ * empty) and the depth ("2", else "3"). The list of links is the derived outline and is ignored.
+ */
+function tocRule(key: KeyGenerator): DeserializerRule {
+  return {
+    deserialize(el: Node, _next, createBlock) {
+      if (!isElement(el) || el.nodeName.toLowerCase() !== "nav" || el.getAttribute("data-rt") !== "toc") return undefined;
+      const title = (el.querySelector("[data-title]")?.textContent ?? "").trim().slice(0, PT_TOC_LIMITS.titleChars);
+      const depth = el.getAttribute("data-depth") === "2" ? "2" : "3";
+      return createBlock({ _type: PT_TYPES.tableOfContents, _key: key(), ...(title ? { title } : {}), depth });
+    },
+  };
+}
+
+/**
+ * RT4: `<figure data-rt="gallery" data-layout>` → the `gallery` block, the inverse of `galleryHtml`
+ * and of V2's own markup. One image per `<li>`: its `<img>` src and alt, and its figcaption as the
+ * caption (≤ 160). A `<li>` with no `<img>` is an image not yet chosen, as V2 reads it. The
+ * layout is a carousel only when it says so.
+ */
+function galleryFigureRule(key: KeyGenerator): DeserializerRule {
+  return {
+    deserialize(el: Node, _next, createBlock) {
+      if (!isElement(el) || el.nodeName.toLowerCase() !== "figure" || el.getAttribute("data-rt") !== "gallery") return undefined;
+      const images = Array.from(el.querySelectorAll("li")).map((li) => {
+        const img = li.querySelector("img");
+        const caption = (li.querySelector("figcaption")?.textContent ?? "").trim().slice(0, PT_GALLERY_LIMITS.captionChars);
+        const frame = img ? readImageFrame(img) : null;
+        return {
+          _key: key(),
+          image: img
+            ? {
+                src: frame?.src ?? (img.getAttribute("src") ?? "").trim(),
+                alt: img.getAttribute("alt") ?? "",
+                ...(frame?.crop ? { crop: frame.crop } : {}),
+                ...(frame?.focal ? { focal: frame.focal } : {}),
+              }
+            : null,
+          ...(caption ? { caption } : {}),
+        };
+      });
+      const layout = el.getAttribute("data-layout") === "carousel" ? "carousel" : "grid";
+      return createBlock({ _type: PT_TYPES.gallery, _key: key(), layout, images });
+    },
+  };
+}
+
+/**
+ * RT3: `<figure data-rt="embed">` → the `embed` block, the inverse of `embedFigureHtml` and of V2's
+ * own `<figure data-rt="embed" data-url data-aspect><a href>title</a></figure>`. Only what is
+ * stored is read: the url (`data-url`, else the link's href), the title (`aria-label` when the
+ * figure has one, which ours always does, else the link's words, as V2 reads it) and the aspect
+ * (V2's four, else 16:9). The frame, the card and `data-provider` are derived and ignored.
+ */
+function embedFigureRule(key: KeyGenerator, schema: PtSchema | undefined): DeserializerRule {
+  const allowed = allowsBlockObject(schema, PT_TYPES.embed);
+  return {
+    deserialize(el: Node, _next, createBlock) {
+      if (!allowed || !isElement(el) || el.nodeName.toLowerCase() !== "figure" || el.getAttribute("data-rt") !== "embed") return undefined;
+      const link = el.querySelector("a[href]");
+      const url = (el.getAttribute("data-url") ?? link?.getAttribute("href") ?? "").trim();
+      const title = (el.hasAttribute("aria-label") ? el.getAttribute("aria-label") ?? "" : (link?.textContent ?? "")).trim().slice(0, EMBED_TITLE_MAX);
+      const aspect = el.getAttribute("data-aspect");
+      return createBlock({ _type: PT_TYPES.embed, _key: key(), url, title, aspect: isEmbedAspect(aspect) ? aspect : "16:9" });
+    },
+  };
+}
+
+/**
+ * RT2: V2's button, `<a data-button="true" href>`, → a `link` annotation with `button: true` (and
+ * the site's class, a new tab and `rel`, as the classed link keeps them). Gated on the schema's
+ * link annotation having the `button` field: a field without the button feature reads it as the
+ * plain link it also is.
+ */
+function buttonLinkRule(key: KeyGenerator, schema: PtSchema | undefined): DeserializerRule {
+  const allowed =
+    !schema || schema.annotations.some((a) => a.name === PT_TYPES.link && (a.fields ?? []).some((f) => f.name === "button"));
+  return {
+    deserialize(el: Node, next) {
+      if (!allowed || !isElement(el) || el.nodeName.toLowerCase() !== "a") return undefined;
+      const flag = el.getAttribute("data-button");
+      const href = el.getAttribute("href")?.trim();
+      if (flag === null || flag === "false" || !href) return undefined;
+      const className = el.getAttribute("class")?.trim();
+      const rel = el.getAttribute("rel")?.trim();
+      const markDef = {
+        _type: PT_TYPES.link,
+        _key: key(),
+        href,
+        button: true,
+        ...(className ? { className } : {}),
+        ...(el.getAttribute("target") === "_blank" ? { target: "_blank" } : {}),
+        ...(rel ? { rel } : {}),
+      };
+      return { _type: "__annotation", markDef, children: next(el.childNodes) } as never;
+    },
+  };
+}
+
 /**
  * `<a class>` → a `link` annotation that keeps the site's class as `className`.
  *
@@ -595,7 +853,7 @@ const PDF_HREF = /\.pdf(?:[?#]|$)/i;
  * and four rules would each have to ask that question. `undefined` hands the node back to the
  * deserialiser untouched, which is what keeps a mid-sentence PDF link a link.
  */
-function mediaRule(key: KeyGenerator, schema: PtSchema | undefined): DeserializerRule {
+function mediaRule(key: KeyGenerator, schema: PtSchema | undefined, onReceipt?: (line: string) => void): DeserializerRule {
   const allows = (name: string): boolean =>
     !schema || schema.blockObjects.some((o) => o.name === name);
   return {
@@ -628,6 +886,15 @@ function mediaRule(key: KeyGenerator, schema: PtSchema | undefined): Deserialize
           ...(caption ? { title: caption } : {}),
         });
       }
+      // RT3: a frame the embed allowlist recognises is an `embed`, in a field that takes one.
+      if (subject === "iframe" && allows(PT_TYPES.embed)) {
+        const src = (el.getAttribute("src") ?? "").trim();
+        if (src && resolveEmbed(src).kind === "frame") {
+          const title = ((el.getAttribute("title") ?? "").trim() || caption).slice(0, EMBED_TITLE_MAX);
+          const aspect = nearestEmbedAspect(el.getAttribute("width"), el.getAttribute("height"));
+          return createBlock({ _type: PT_TYPES.embed, _key: key(), url: src, title, aspect });
+        }
+      }
       // Image and embed keep the GENERIC block object under their existing `builtin:` keys —
       // thousands of stored values already spell them that way, and nothing is gained by a
       // migration when the renderer reads both shapes from the same fields.
@@ -636,6 +903,18 @@ function mediaRule(key: KeyGenerator, schema: PtSchema | undefined): Deserialize
       if (!src) return undefined;
       if (subject === "iframe") {
         const title = (el.getAttribute("title") ?? "").trim() || caption;
+        // Any other frame keeps the legacy, read-only `builtin:video` path: a working Calendly-like
+        // embed must not turn into a link (R15). Recorded, so the allowlist can grow from evidence.
+        // Only a frame the allowlist does not know is evidence; a known one here just met a field without the embed.
+        if (resolveEmbed(src).kind !== "frame") {
+          let host = "invalid";
+          try {
+            host = new URL(src).hostname;
+          } catch {
+            /* an unparsable src is recorded as such */
+          }
+          onReceipt?.(`embed.legacy_iframe:${host}`);
+        }
         return createBlock({
           _type: PT_TYPES.block,
           _key: key(),
@@ -643,14 +922,18 @@ function mediaRule(key: KeyGenerator, schema: PtSchema | undefined): Deserialize
           fields: { url: src, ...(title ? { title } : {}) },
         });
       }
+      // RT9: an image we drew with a crop or focal point carries what was stored back.
+      const frame = readImageFrame(el);
       return createBlock({
         _type: PT_TYPES.block,
         _key: key(),
         schemaKey: BUILTIN_IMAGE_KEY,
         fields: {
-          src,
+          src: frame.src ?? src,
           alt: (el.getAttribute("alt") ?? "").trim(),
           ...(caption ? { caption } : {}),
+          ...(frame.crop ? { crop: frame.crop } : {}),
+          ...(frame.focal ? { focal: frame.focal } : {}),
         },
       });
     },
@@ -792,6 +1075,104 @@ const allowsStyled = (schema: PtSchema | undefined): boolean =>
 const allowsBlockObject = (schema: PtSchema | undefined, name: string): boolean =>
   !schema || schema.blockObjects.some((o) => o.name === name);
 
+const words = (s: string): string => s.replace(/\u200b/g, "").replace(/\s+/g, " ").trim();
+
+/**
+ * RT9: each top-level h2/h3 of the page being read, in order, with its own `id` ("" when none or
+ * not a plain id). The words exclude a footnote marker, as the stored heading's do.
+ */
+const withHeadingIds =
+  (parse: (html: string) => Document, into: { text: string; id: string }[]) =>
+  (html: string): Document => {
+    const doc = parse(html);
+    for (const h of Array.from(doc.body.querySelectorAll(":scope > h2, :scope > h3"))) {
+      const copy = h.cloneNode(true) as Element;
+      copy.querySelectorAll("sup").forEach((s) => s.remove());
+      const id = h.getAttribute("id") ?? "";
+      into.push({ text: words(copy.textContent ?? ""), id: isHeadingAnchor(id) ? id : "" });
+    }
+    return doc;
+  };
+
+/**
+ * RT9: an imported heading keeps its page's id as `anchor`, so the site's own links to it keep
+ * working, but only when it differs from the id its words make: our own markup reads back with
+ * no anchor at all, and a heading's id follows its words unless the page said otherwise.
+ */
+function keepHeadingIds(value: PortableText, ids: { text: string; id: string }[]): PortableText {
+  if (!ids.some((h) => h.id)) return value;
+  const heads = value.filter(
+    (n): n is PortableTextBlock => n._type === "block" && !(n as PortableTextBlock).listItem && ((n as PortableTextBlock).style === "h2" || (n as PortableTextBlock).style === "h3"),
+  );
+  let at = 0;
+  const marked: PortableTextBlock[] = [];
+  for (const head of heads) {
+    const text = words(head.children.map((c) => (typeof (c as { text?: unknown }).text === "string" ? (c as { text: string }).text : "")).join(""));
+    while (at < ids.length && ids[at].text !== text) at++;
+    if (at >= ids.length) break;
+    if (ids[at].id) {
+      head.anchor = ids[at].id;
+      marked.push(head);
+    }
+    at++;
+  }
+  // The ids the words alone would make: an anchor equal to its heading's is not kept.
+  const set = new Set<unknown>(marked);
+  const plain = value.map((n) => (set.has(n) ? { ...n, anchor: undefined } : n));
+  const auto = outlineOf(plain).nodes;
+  value.forEach((n, i) => {
+    if (set.has(n) && auto.get(plain[i]) === (n as PortableTextBlock).anchor) delete (n as PortableTextBlock).anchor;
+  });
+  return value;
+}
+
+const allowsInlineObject = (schema: PtSchema | undefined, name: string): boolean =>
+  !schema || schema.inlineObjects.some((o) => o.name === name);
+
+/**
+ * RT6: V2's end list (`<ol data-rt="footnotes">`, or a legacy `ol.cs-footnotes`) is derived, so it
+ * is dropped before the deserialiser sees it (V2 `hydrate`). A legacy marker that kept its note only
+ * in the list (`<sup id="fnref-N">`, no `data-note`) is lent that note first.
+ */
+const withFootnoteNotes =
+  (parse: (html: string) => Document) =>
+  (html: string): Document => {
+    const doc = parse(html);
+    const lists = Array.from(doc.body.querySelectorAll(":scope > ol[data-rt=footnotes], :scope > ol.cs-footnotes"));
+    if (lists.length === 0) return doc;
+    const legacy: Record<number, string> = {};
+    for (const ol of lists) {
+      for (const li of Array.from(ol.querySelectorAll(":scope > li[id^='fn-']"))) {
+        const n = Number(li.id.slice(3));
+        const copy = li.cloneNode(true) as Element;
+        copy.querySelectorAll("a[href^='#fnref-']").forEach((a) => a.remove());
+        if (n) legacy[n] = (copy.textContent ?? "").replace(/\u200b/g, "").trim();
+      }
+      ol.remove();
+    }
+    for (const sup of Array.from(doc.body.querySelectorAll("sup[data-rt=fn], sup[id^='fnref-']"))) {
+      if (sup.hasAttribute("data-note")) continue;
+      const n = Number(sup.id.replace("fnref-", "")) || Number(sup.textContent) || 0;
+      sup.setAttribute("data-note", legacy[n] ?? "");
+    }
+    return doc;
+  };
+
+/**
+ * RT6: a footnote marker, ours or V2's `<sup data-rt="fn" data-note>` (or a legacy `<sup
+ * id="fnref-N">`) → the `footnote` inline object. Only the note is read: the number, the ids and
+ * the link inside are derived.
+ */
+function footnoteRule(key: KeyGenerator): DeserializerRule {
+  return {
+    deserialize(el: Node) {
+      if (!isElement(el) || el.nodeName.toLowerCase() !== "sup") return undefined;
+      if (el.getAttribute("data-rt") !== "fn" && !(el.getAttribute("id") ?? "").startsWith("fnref-")) return undefined;
+      return { _type: PT_TYPES.footnote, _key: key(), note: el.getAttribute("data-note") ?? "" } as never;
+    },
+  };
+}
+
 const isElement = (n: unknown): n is Element =>
   !!n && typeof n === "object" && (n as Node).nodeType === 1;
 
@@ -823,11 +1204,18 @@ const withoutBlockWhitespace =
 export function htmlToPortableText(html: string, opts: IngestOptions = {}): PortableText {
   if (typeof html !== "string" || html.trim() === "") return [];
   const key = opts.keyGenerator ?? randomKey;
+  const headingIds: { text: string; id: string }[] = [];
   const parse =
     opts.parseHtml ?? (typeof DOMParser !== "undefined" ? (markup: string) => new DOMParser().parseFromString(markup, "text/html") : undefined);
   const converted = ptHtmlToPortableText(html, {
     // The browser's own parser when none is passed (Node must pass one), so both drop the same whitespace.
-    ...(parse ? { parseHtml: withoutBlockWhitespace(parse) } : {}),
+    ...(parse
+      ? {
+          parseHtml: ((p) => (opts.headingAnchors ? withHeadingIds(p, headingIds) : p))(
+            allowsInlineObject(opts.schema, PT_TYPES.footnote) ? withFootnoteNotes(withoutBlockWhitespace(parse)) : withoutBlockWhitespace(parse),
+          ),
+        }
+      : {}),
     // `ptSchemaFor` returns a schema DEFINITION — the same object `EditorProvider` takes, so
     // the editor and this filter are literally configured from one value. The deserialiser
     // wants the COMPILED form, so the compile happens here rather than at every call site.
@@ -845,19 +1233,32 @@ export function htmlToPortableText(html: string, opts: IngestOptions = {}): Port
      * accepts only images.
      */
     rules: [
+      // First: a marker's content is the site's, never this field's text.
+      ...(opts.siteParts || allowsBlockObject(opts.schema, PT_TYPES.block) ? [siteRule(key)] : []),
       ...(allowsBlockObject(opts.schema, PT_TYPES.table)
         ? [tableFigureRule(key, (cell) => htmlToPortableText(cell, opts)), tableRule(key, (cell) => htmlToPortableText(cell, opts))]
         : []),
+      ...(allowsInlineObject(opts.schema, PT_TYPES.footnote) ? [footnoteRule(key)] : []),
       ...(allowsBlockObject(opts.schema, PT_TYPES.code) ? [codeRule(key)] : []),
       ...(allowsBlockObject(opts.schema, PT_TYPES.pullQuote) ? [pullQuoteRule(key)] : []),
       ...(allowsBlockObject(opts.schema, PT_TYPES.keyTakeaways) ? [keyTakeawaysRule(key)] : []),
-      mediaRule(key, opts.schema),
+      // Before the media rule, which would take the iframe out of our own embed figure (and an image out of a gallery).
+      ...(allowsBlockObject(opts.schema, PT_TYPES.gallery) ? [galleryFigureRule(key)] : []),
+      ...(allowsBlockObject(opts.schema, PT_TYPES.tableOfContents) ? [tocRule(key)] : []),
+      referenceRule(key, opts.schema),
+      ...(allowsBlockObject(opts.schema, PT_TYPES.columns) ? [columnsRule(key, (inner) => htmlToPortableText(inner, opts))] : []),
+      embedFigureRule(key, opts.schema),
+      mediaRule(key, opts.schema, opts.onReceipt),
+      checklistItemRule(opts.schema),
+      ownDecoratorRule(opts.schema),
       ...(allowsStyled(opts.schema) ? [styledRule(key)] : []),
+      buttonLinkRule(key, opts.schema),
       ...(allowsLinks(opts.schema) ? [classedLinkRule(key)] : []),
     ],
     keyGenerator: key,
   });
-  return normalise(converted, key);
+  const value = normalise(converted, key);
+  return opts.headingAnchors ? keepHeadingIds(value, headingIds) : value;
 }
 
 /**

package/src/plain.ts

@@ -1,6 +1,8 @@
 import { isPortableTextBlock, isPortableTextSpan, PT_TYPES, type PortableText, type PortableTextTable } from "./types";
 import { migrateTable, tableGroupLayout } from "./table";
 import { keyTakeawaysView } from "./key-takeaways";
+import { galleryView } from "./gallery";
+import { columnsView } from "./columns";
 import { componentCopyLines, detachedComponentCopy } from "./component-copy";
 
 /**
@@ -49,6 +51,20 @@ export function portableTextToPlainText(value: PortableText | null | undefined):
       lines.push(...(title ? [title] : []), ...points);
       continue;
     }
+    /** A gallery's captions are words a reader sees; its images are not. */
+    if (node._type === PT_TYPES.gallery) {
+      lines.push(...galleryView(node).images.map((x) => x.caption).filter(Boolean));
+      continue;
+    }
+    /** A column's text is words a reader sees, in column order (a legacy `builtin:columns` too). */
+    const cols = columnsView(node);
+    if (cols) {
+      for (const c of cols.columns) {
+        const text = portableTextToPlainText(c.content);
+        if (text) lines.push(text);
+      }
+      continue;
+    }
     const table = migrateTable(node);
     if (table) {
       lines.push(...tableLines(table));

package/src/read.ts

@@ -3,6 +3,7 @@ import { lexicalToPortableText } from "./from-lexical";
 import { migrateTable } from "./table";
 import { detachedComponentCopy, type DetachedComponentCopy } from "./component-copy";
 import { randomKey, type KeyGenerator } from "./key";
+import type { EmbedCard, ResolvedAuthor, ResolvedPost, RichTextEmbeds, RichTextResolved } from "@bettercms-ai/types";
 
 /**
  * THE reader. One function, every stored shape, and it never throws.
@@ -233,6 +234,31 @@ export function readStructuredContent(props: unknown): PortableText | null {
   return arr && arr.length > 0 ? (arr as PortableText) : null;
 }
 
+/**
+ * The embeds a delivered value carries beside its html (`RichTextEmbeds`), from the same envelope
+ * `readStructuredContent` reads; undefined when there are none. Delivered data is checked, not
+ * trusted: a card keeps only string fields.
+ */
+export function readStructuredEmbeds(props: unknown): RichTextEmbeds | undefined {
+  if (!isRecord(props)) return undefined;
+  const candidate = props.content ?? props.value;
+  const raw = isRecord(candidate) ? candidate.embeds : undefined;
+  if (!isRecord(raw)) return undefined;
+  const cards: Record<string, EmbedCard> = {};
+  if (isRecord(raw.cards)) {
+    for (const [url, c] of Object.entries(raw.cards)) {
+      if (!isRecord(c) || typeof c.text !== "string" || typeof c.authorName !== "string") continue;
+      cards[url] = {
+        text: c.text,
+        authorName: c.authorName,
+        ...(typeof c.authorUrl === "string" ? { authorUrl: c.authorUrl } : {}),
+        ...(typeof c.date === "string" ? { date: c.date } : {}),
+      };
+    }
+  }
+  return { cards };
+}
+
 /** The `component` block an embed in rich text stands for. */
 export interface EmbeddedComponentBlock {
   type: "component";
@@ -272,3 +298,40 @@ export function embeddedComponentRender(
   const copy = detachedComponentCopy(value);
   return copy ? { kind: "copy", copy } : null;
 }
+
+const strings = <K extends string>(o: Record<string, unknown>, keys: readonly K[]): Partial<Record<K, string>> =>
+  Object.fromEntries(keys.filter((k) => typeof o[k] === "string").map((k) => [k, o[k]])) as Partial<Record<K, string>>;
+
+/**
+ * RT8 (G-RT8): the entries a DELIVERED value's author bios and related posts point at, which the
+ * delivery puts beside the value (`value.resolved`); undefined when there are none. Never read from
+ * a stored envelope: nothing stores it. Checked, not trusted: a post keeps only its string fields
+ * and needs a title and url, an author a name, and `related` only lists of ids.
+ */
+export function readStructuredResolved(props: unknown): RichTextResolved | undefined {
+  if (!isRecord(props)) return undefined;
+  const candidate = props.content ?? props.value;
+  const raw = isRecord(candidate) ? candidate.resolved : undefined;
+  if (!isRecord(raw)) return undefined;
+  const posts: Record<string, ResolvedPost> = {};
+  if (isRecord(raw.posts)) {
+    for (const [id, p] of Object.entries(raw.posts)) {
+      if (isRecord(p) && typeof p.title === "string" && typeof p.url === "string") {
+        posts[id] = { title: p.title, url: p.url, ...strings(p, ["cover", "category"] as const) };
+      }
+    }
+  }
+  const authors: Record<string, ResolvedAuthor> = {};
+  if (isRecord(raw.authors)) {
+    for (const [id, a] of Object.entries(raw.authors)) {
+      if (isRecord(a) && typeof a.name === "string") authors[id] = { name: a.name, ...strings(a, ["avatar", "bio", "url"] as const) };
+    }
+  }
+  const related: Record<string, string[]> = {};
+  if (isRecord(raw.related)) {
+    for (const [key, ids] of Object.entries(raw.related)) {
+      if (Array.isArray(ids)) related[key] = ids.filter((id): id is string => typeof id === "string");
+    }
+  }
+  return { posts, authors, related };
+}