@bettercms-ai/richtext 0.12.0 → 0.14.0

package/src/html.ts

@@ -16,6 +16,9 @@ import { safeHref } from "./href";
 import { isStyledTag, styledAttrEntries } from "./styled";
 import { richTextImageLayout } from "./image-layout";
 import { keyTakeawaysView } from "./key-takeaways";
+import { GALLERY_STYLES, galleryView } from "./gallery";
+import { outlineOf, tocTree, tocView, type OutlineItem } from "./toc";
+import { isEmbedAspect, resolveEmbed, type EmbedCard, type EmbedRender } from "./embed";
 
 /**
  * Portable Text → HTML.
@@ -420,12 +423,97 @@ function keyTakeawaysHtml(value: unknown, classes: HtmlClasses): string {
   return `<aside data-rt="takeaways">${title ? `<p data-title>${esc(title)}</p>` : ""}<ul${cls(classes, "ul")}>${items}</ul></aside>`;
 }
 
+/**
+ * RT4: a gallery, V2's `<figure data-rt="gallery" data-layout><ul><li><figure><img><figcaption>`.
+ * The layout needs no script: a grid, or a carousel that is a scroll-snap row (swipe, scroll, or
+ * the keyboard once the row has focus). The styles are inline because a hosted page has no
+ * stylesheet for them; the ingest reads only the markup back. `src` comes last on the `<img>`, as
+ * React writes a lazy image's, so both renderers give the same bytes (R6).
+ */
+function galleryHtml(value: unknown): string {
+  const { layout, images } = galleryView(value);
+  const s = GALLERY_STYLES[layout];
+  const items = images
+    .map((x) => `<li style="${s.li}"><figure style="${s.figure}"><img alt="${esc(x.alt)}" style="${s.img}" loading="lazy" src="${esc(x.src)}">${
+      x.caption ? `<figcaption>${esc(x.caption)}</figcaption>` : ""
+    }</figure></li>`)
+    .join("");
+  const focus = layout === "carousel" ? ' tabindex="0"' : "";
+  return `<figure data-rt="gallery" data-layout="${layout}"><ul style="${s.ul}"${focus}>${items}</ul></figure>`;
+}
+
+/**
+ * RT5: the outline of the value being rendered. Set by `portableTextToHtml` for a whole-value
+ * render and restored after it (a render is synchronous, and a table cell renders inside another
+ * one); an inline render (a cell, a heading field) keeps the value's outline it sits in.
+ */
+let outline: { items: OutlineItem[]; nodes: WeakMap<object, string> } = { items: [], nodes: new WeakMap() };
+
+/** An h2 or h3 with its outline id, so a table of contents (or any link) can point at it. */
+const headingWithId = (tag: "h2" | "h3", classes: HtmlClasses) =>
+  ({ value, children }: { value: object; children?: string }) => {
+    const id = outline.nodes.get(value);
+    return `<${tag}${id ? ` id="${esc(id)}"` : ""}${cls(classes, tag)}>${children ?? ""}</${tag}>`;
+  };
+
+/**
+ * RT5: a table of contents, V2's `<nav data-rt="toc" data-depth aria-label><p data-title>…</p>
+ * <ol><li><a href="#id">…</a><ol>…</ol></li></ol></nav>`, listing the outline of the value it is in.
+ */
+function tocHtml(value: unknown): string {
+  const { title, label, depth } = tocView(value);
+  const link = (o: OutlineItem) => `<a href="#${esc(o.id)}">${esc(o.text)}</a>`;
+  const items = tocTree(outline.items, depth)
+    .map((e) => `<li>${link(e)}${e.children.length ? `<ol>${e.children.map((c) => `<li>${link(c)}</li>`).join("")}</ol>` : ""}</li>`)
+    .join("");
+  return `<nav data-rt="toc" data-depth="${depth}" aria-label="${esc(label)}">${title ? `<p data-title>${esc(title)}</p>` : ""}<ol>${items}</ol></nav>`;
+}
+
+/**
+ * An X post's card, from the oEmbed data the server fetched (plain strings, escaped here; never
+ * X's markup). Always the blockquote X's widget recognises: it reads as a quote without
+ * JavaScript, and on a site that opted in, the page's one widgets.js (next `BcmsEmbedScripts`)
+ * upgrades it in place. No script here: the html is a function of the value and its cards only,
+ * so turning the opt-in on or off re-bakes no html cache.
+ */
+function xCardHtml(url: string, card: EmbedCard): string {
+  return `<blockquote class="twitter-tweet bcms-embed-card" cite="${esc(url)}"><p>${esc(card.text)}</p>&mdash; ${esc(card.authorName)} <a href="${esc(url)}">${esc(card.date || "View on X")}</a></blockquote>`;
+}
+
+/**
+ * RT3: an embed, V2's `<figure data-rt="embed" data-url data-provider data-aspect>` around what
+ * its url resolves to (`resolveEmbed`): the provider's frame, an audio player, an X card, or a
+ * link card. `aria-label` carries the stored title, present even when empty, so the ingest reads
+ * back exactly what was stored (V2's own markup has the title in the link instead).
+ */
+function embedFigureHtml(value: unknown, embeds: EmbedRender): string {
+  const v = (value ?? {}) as { url?: unknown; title?: unknown; aspect?: unknown };
+  const url = typeof v.url === "string" ? v.url.trim() : "";
+  const aspect = isEmbedAspect(v.aspect) ? v.aspect : "16:9";
+  const title = typeof v.title === "string" ? v.title.trim() : "";
+  const r = resolveEmbed(url, aspect);
+  const label = title || r.label;
+  let body: string;
+  if (r.kind === "frame") body = `<iframe src="${esc(r.src)}" title="${esc(label)}" width="${r.width}" height="${r.height}"${IFRAME_ATTRS}></iframe>`;
+  else if (r.kind === "audio") body = `<audio controls preload="none" src="${esc(r.src)}" title="${esc(label)}"></audio>`;
+  // Cards are keyed by the canonical url (`r.url`), so a post pasted two ways finds its card.
+  else if (r.kind === "card" && embeds.cards?.[r.url]) body = xCardHtml(r.url, embeds.cards[r.url]);
+  // A link card: no rule frames this url, or it is an X post with no card data for this render.
+  else body = r.url ? `<a href="${esc(r.url)}">${esc(label)}</a>` : "";
+  return `<figure data-rt="embed" data-url="${esc(url)}" data-provider="${r.provider}" data-aspect="${aspect}" aria-label="${esc(title)}">${body}</figure>`;
+}
+
 export interface RenderOptions {
   /**
    * Render a single value with no wrapping `<p>`, for a field whose consumer supplies its own
    * heading or button. Mirrors `allowMultipleParagraphs: false` and the old `toInlineHtml`.
    */
   inline?: boolean;
+  /**
+   * RT3: what an embed needs that the value does not hold: the X cards the server fetched for this
+   * render (by url), and the site's opt-in to X's script. Absent: every X post is a link card.
+   */
+  embeds?: EmbedRender;
   /**
    * The site's own class per tag, for a value derived FROM that site.
    *
@@ -480,6 +568,17 @@ const bindingAttrs = (base: string, key: unknown): string =>
     ? ` data-bcms-field="${esc(base)}#${esc(key)}" data-bcms-kind="richtext"`
     : "";
 
+/**
+ * A checklist item, as V2 stores it (`<li data-checked="true|false">`), with a disabled checkbox in
+ * front of the words so a page without the V2 stylesheet still shows the box and its state, and a
+ * screen reader announces it. The ingest reads `data-checked` back and skips the box.
+ */
+const checkItemHtml = (classes: HtmlClasses, value: { checked?: unknown }, children: string | undefined, address: string): string => {
+  const checked = value.checked === true;
+  // `disabled` before `type`: the order React writes an input's attributes in, so the two renderers match (R6).
+  return `<li${cls(classes, "li")}${address} data-checked="${checked}"><input disabled type="checkbox"${checked ? " checked" : ""}>${children ?? ""}</li>`;
+};
+
 /**
  * `keys` is the set of `_key`s the value actually carries: to-html invents one (`li-0-1`) for an
  * item that has none, and an invented key is an address that resolves to nothing.
@@ -488,14 +587,26 @@ const buildComponents = (
   classes: HtmlClasses = {},
   binding?: { base: string; keys: ReadonlySet<unknown> },
   renderBlock?: RenderOptions["renderBlock"],
+  embeds: EmbedRender = {},
 ): Partial<PortableTextHtmlComponents> => ({
-  block: tagOverrides(classes, STYLE_TAGS),
-  list: tagOverrides(classes, { bullet: "ul", number: "ol", check: "ul" }),
+  block: { ...tagOverrides(classes, STYLE_TAGS), h2: headingWithId("h2", classes), h3: headingWithId("h3", classes) },
+  list: {
+    ...tagOverrides(classes, { bullet: "ul", number: "ol" }),
+    // A checklist is V2's `<ul data-task="true">`, dressed or not: it is not a bullet list.
+    check: ({ children }: { children?: string }) => `<ul data-task="true"${cls(classes, "ul")}>${children ?? ""}</ul>`,
+  },
   // With a base, every item carries its own address ON the `<li>` — see `bindingBase`.
   listItem: binding
-    ? ({ value, children }: { value: { _key?: unknown }; children?: string }) =>
-        `<li${cls(classes, "li")}${binding.keys.has(value._key) ? bindingAttrs(binding.base, value._key) : ""}>${children ?? ""}</li>`
-    : tagOverrides(classes, { bullet: "li", number: "li", check: "li" }),
+    ? ({ value, children }: { value: { _key?: unknown; listItem?: unknown; checked?: unknown }; children?: string }) => {
+        const address = binding.keys.has(value._key) ? bindingAttrs(binding.base, value._key) : "";
+        return value.listItem === "check"
+          ? checkItemHtml(classes, value, children, address)
+          : `<li${cls(classes, "li")}${address}>${children ?? ""}</li>`;
+      }
+    : {
+        ...tagOverrides(classes, { bullet: "li", number: "li" }),
+        check: ({ value, children }) => checkItemHtml(classes, value as { checked?: unknown }, children, ""),
+      },
   types: {
     [PT_TYPES.divider]: () => "<hr>",
     [PT_TYPES.table]: ({ value }) => tableHtml(value as PortableTextTable),
@@ -510,6 +621,9 @@ const buildComponents = (
     [PT_TYPES.file]: ({ value }) => fileHtml(value, classes),
     [PT_TYPES.pullQuote]: ({ value }) => pullQuoteHtml(value, classes),
     [PT_TYPES.keyTakeaways]: ({ value }) => keyTakeawaysHtml(value, classes),
+    [PT_TYPES.embed]: ({ value }) => embedFigureHtml(value, embeds),
+    [PT_TYPES.gallery]: ({ value }) => galleryHtml(value),
+    [PT_TYPES.tableOfContents]: ({ value }) => tocHtml(value),
     /**
      * A generic embedded block. A block whose payload this serialiser can READ renders as
      * content; one carrying a reference it would have to resolve against a project renders as a
@@ -557,9 +671,11 @@ const buildComponents = (
      * renders the LABEL rather than deleting it — a blocked link must not lose the words.
      */
     [PT_TYPES.link]: ({ value, children }) => {
-      const v = (value ?? {}) as { href?: unknown; target?: unknown; rel?: unknown; className?: unknown };
+      const v = (value ?? {}) as { href?: unknown; target?: unknown; rel?: unknown; className?: unknown; button?: unknown };
       const href = typeof v.href === "string" ? safeHref(v.href) : null;
       if (!href) return children;
+      // RT2: a button is a link the author marked as one; V2's `data-button` is the site's style hook.
+      const button = v.button === true ? ' data-button="true"' : "";
       // `rel` is author data too (the importer keeps `prev`/`next` on pagination links), so only
       // plain link-type tokens pass; a new tab always adds noopener + noreferrer.
       const rel = new Set(
@@ -573,7 +689,7 @@ const buildComponents = (
       const relAttr = rel.size ? ` rel="${esc([...rel].join(" "))}"` : "";
       // The link's own class (kept on ingest) wins over the caller's default: it IS the site.
       const own = typeof v.className === "string" && v.className ? ` class="${esc(v.className)}"` : "";
-      return `<a${own || cls(classes, "a")} href="${esc(href)}"${target}${relAttr}>${children}</a>`;
+      return `<a${own || cls(classes, "a")}${button} href="${esc(href)}"${target}${relAttr}>${children}</a>`;
     },
     /**
      * The site's own inline markup, written back out as itself.
@@ -617,12 +733,25 @@ export function portableTextToHtml(
   opts: RenderOptions = {},
 ): string {
   if (!Array.isArray(value) || value.length === 0) return "";
+  if (opts.inline) return render(value, opts);
+  const outer = outline;
+  outline = outlineOf(value);
+  try {
+    return render(value, opts);
+  } finally {
+    outline = outer;
+  }
+}
+
+function render(value: PortableText, opts: RenderOptions): string {
   const components =
-    opts.classes || opts.renderBlock ? buildComponents(opts.classes, undefined, opts.renderBlock) : defaultComponents;
+    opts.classes || opts.renderBlock || opts.embeds
+      ? buildComponents(opts.classes, undefined, opts.renderBlock, opts.embeds)
+      : defaultComponents;
 
   if (opts.bindingBase) {
     const base = opts.bindingBase;
-    const bound = buildComponents(opts.classes, { base, keys: new Set(value.map((n) => n._key)) }, opts.renderBlock);
+    const bound = buildComponents(opts.classes, { base, keys: new Set(value.map((n) => n._key)) }, opts.renderBlock, opts.embeds);
     const out: string[] = [];
     for (let i = 0; i < value.length; ) {
       const node = value[i];

package/src/index.ts

@@ -6,10 +6,14 @@ export * from "./from-lexical";
 export * from "./plain";
 export * from "./table";
 export * from "./key-takeaways";
+export * from "./gallery";
+export * from "./toc";
 export * from "./styled";
 export * from "./read";
 export * from "./component-copy";
 export * from "./accept";
 export * from "./features";
+export * from "./write-refusals";
 export * from "./video";
+export * from "./embed";
 export * from "./image-layout";

package/src/ingest.ts

@@ -3,6 +3,7 @@ import { compileSchema } from "@portabletext/schema";
 import {
   BUILTIN_EMBED_KEY,
   BUILTIN_IMAGE_KEY,
+  BUILTIN_SITE_KEY,
   PT_TYPES,
   type PortableText,
   type PortableTextBlock,
@@ -15,8 +16,11 @@ import {
 import { randomKey, isValidKey, type KeyGenerator } from "./key";
 import { safeHref } from "./href";
 import { BARE_IS_DESIGN, isStyledTag, styledAttrsOf, type StyledMarkDef } from "./styled";
+import { EMBED_TITLE_MAX, isEmbedAspect, nearestEmbedAspect, resolveEmbed } from "./embed";
 import { cellTypeOf, columnOf, designOf, isColumnType, isValueType, jsonLdOf, PT_TABLE_LIMITS, typedCellText } from "./table";
 import { PT_KEY_TAKEAWAYS_LIMITS } from "./key-takeaways";
+import { PT_GALLERY_LIMITS } from "./gallery";
+import { PT_TOC_LIMITS } from "./toc";
 import type { PtSchema } from "./schema";
 
 /**
@@ -48,6 +52,17 @@ export interface IngestOptions {
   /** What the field accepts. Anything outside it produces no node. */
   schema?: PtSchema;
   keyGenerator?: KeyGenerator;
+  /**
+   * Keep parts the site renders (`builtin:site` markers) where the schema has no generic block.
+   * Only the marker is admitted: the schema still decides every other node.
+   */
+  siteParts?: boolean;
+  /**
+   * Called once per fact an import receipt records, today `embed.legacy_iframe:<host>`: an
+   * `<iframe>` the embed allowlist does not frame, kept on the legacy read-only path. The evidence
+   * the allowlist grows from.
+   */
+  onReceipt?: (line: string) => void;
 }
 
 /**
@@ -460,6 +475,29 @@ const codeLanguage = (pre: Element): string | undefined =>
  * deserialiser's default flattens a code block to a paragraph with a `code` mark and its
  * newlines collapsed: every imported code fence became one line of inline code.
  */
+/**
+ * The marker a part the SITE renders is written as (`<div class="bcms-block" data-bcms-schema-key=
+ * "builtin:site">`, @see BUILTIN_SITE_KEY) → that locked block, never its contents as paragraphs.
+ * The repo source and the card's label ride on `data-bcms-site-source` / `data-bcms-site-label`,
+ * which only the source lane writes; a rendered marker carries neither and still comes back locked.
+ */
+function siteRule(key: KeyGenerator): DeserializerRule {
+  return {
+    deserialize(el: Node, _next, createBlock) {
+      if (!isElement(el) || el.nodeName.toLowerCase() !== "div" || el.getAttribute("data-bcms-schema-key") !== BUILTIN_SITE_KEY) return undefined;
+      const source = el.getAttribute("data-bcms-site-source") ?? "";
+      const label = el.getAttribute("data-bcms-site-label") ?? "";
+      return createBlock({
+        _type: PT_TYPES.block,
+        _key: key(),
+        schemaKey: BUILTIN_SITE_KEY,
+        schemaLabel: label || "Site component",
+        fields: source ? { source } : {},
+      });
+    },
+  };
+}
+
 function codeRule(key: KeyGenerator): DeserializerRule {
   return {
     deserialize(el: Node, _next, createBlock) {
@@ -513,6 +551,156 @@ function styledRule(key: KeyGenerator): DeserializerRule {
   };
 }
 
+/**
+ * The two decorators the renderer writes in markup the standard rules do not read back (R1):
+ * - `<mark>`: the deserialiser names it `mark`, the schema calls it `highlight`, so it was
+ *   dropped on every round trip;
+ * - `<span style="text-decoration:underline">`: what `portableTextToHtml` emits for `underline`,
+ *   which the styled rule then recorded as the site's own design instead of the decorator.
+ * Bare only: a classed `<mark>` or a span with anything else on it is still the styled rule's.
+ * Gated on the schema, because a custom rule's output is never filtered by it.
+ */
+function ownDecoratorRule(schema: PtSchema | undefined): DeserializerRule {
+  const allows = (name: string) => !schema || schema.decorators.some((d) => d.name === name);
+  return {
+    deserialize(el: Node, next) {
+      if (!isElement(el)) return undefined;
+      const tag = el.nodeName.toLowerCase();
+      const attrs = el.getAttributeNames();
+      if (tag === "mark" && attrs.length === 0 && allows("highlight")) {
+        return { _type: "__decorator", name: "highlight", children: next(el.childNodes) } as never;
+      }
+      const style = el.getAttribute("style")?.replace(/\s+/g, "").replace(/;$/, "").toLowerCase();
+      if (tag === "span" && attrs.length === 1 && style === "text-decoration:underline" && allows("underline")) {
+        return { _type: "__decorator", name: "underline", children: next(el.childNodes) } as never;
+      }
+      return undefined;
+    },
+  };
+}
+
+/**
+ * A V2 checklist item, `<ul data-task="true"><li data-checked="true|false">`, read as
+ * `listItem: "check"` + `checked`; a nested list's depth is added when it is flattened. The box in front of the words (the
+ * renderer's disabled checkbox) is skipped: the state is `data-checked`, never the input. Gated on
+ * the schema, because a custom rule's output is never filtered by it: a field without the
+ * checklist falls through to the standard rule and reads the items as bullets.
+ */
+function checklistItemRule(schema: PtSchema | undefined): DeserializerRule {
+  const allowed = !schema || schema.lists.some((l) => l.name === "check");
+  return {
+    deserialize(el: Node, next) {
+      if (!allowed || !isElement(el) || el.nodeName.toLowerCase() !== "li") return undefined;
+      const list = el.parentNode;
+      if (!list || !isElement(list) || list.nodeName.toLowerCase() !== "ul" || !list.hasAttribute("data-task")) return undefined;
+      const words = Array.from(el.childNodes).filter((n) => !(isElement(n) && n.nodeName.toLowerCase() === "input"));
+      return {
+        _type: "block",
+        style: "normal",
+        markDefs: [],
+        listItem: "check",
+        // 1, as the standard list rule writes it: flattening a nested list adds the depth.
+        level: 1,
+        checked: el.getAttribute("data-checked") === "true",
+        children: next(words as never),
+      } as never;
+    },
+  };
+}
+
+/**
+ * RT5: `<nav data-rt="toc" data-depth>` → the `tableOfContents` block, the inverse of `tocHtml`
+ * and of V2's own markup. Only what is stored is read: the title (`[data-title]`, ≤ 40, none when
+ * empty) and the depth ("2", else "3"). The list of links is the derived outline and is ignored.
+ */
+function tocRule(key: KeyGenerator): DeserializerRule {
+  return {
+    deserialize(el: Node, _next, createBlock) {
+      if (!isElement(el) || el.nodeName.toLowerCase() !== "nav" || el.getAttribute("data-rt") !== "toc") return undefined;
+      const title = (el.querySelector("[data-title]")?.textContent ?? "").trim().slice(0, PT_TOC_LIMITS.titleChars);
+      const depth = el.getAttribute("data-depth") === "2" ? "2" : "3";
+      return createBlock({ _type: PT_TYPES.tableOfContents, _key: key(), ...(title ? { title } : {}), depth });
+    },
+  };
+}
+
+/**
+ * RT4: `<figure data-rt="gallery" data-layout>` → the `gallery` block, the inverse of `galleryHtml`
+ * and of V2's own markup. One image per `<li>`: its `<img>` src and alt, and its figcaption as the
+ * caption (≤ 160). A `<li>` with no `<img>` is an image not yet chosen, as V2 reads it. The
+ * layout is a carousel only when it says so.
+ */
+function galleryFigureRule(key: KeyGenerator): DeserializerRule {
+  return {
+    deserialize(el: Node, _next, createBlock) {
+      if (!isElement(el) || el.nodeName.toLowerCase() !== "figure" || el.getAttribute("data-rt") !== "gallery") return undefined;
+      const images = Array.from(el.querySelectorAll("li")).map((li) => {
+        const img = li.querySelector("img");
+        const caption = (li.querySelector("figcaption")?.textContent ?? "").trim().slice(0, PT_GALLERY_LIMITS.captionChars);
+        return {
+          _key: key(),
+          image: img ? { src: (img.getAttribute("src") ?? "").trim(), alt: img.getAttribute("alt") ?? "" } : null,
+          ...(caption ? { caption } : {}),
+        };
+      });
+      const layout = el.getAttribute("data-layout") === "carousel" ? "carousel" : "grid";
+      return createBlock({ _type: PT_TYPES.gallery, _key: key(), layout, images });
+    },
+  };
+}
+
+/**
+ * RT3: `<figure data-rt="embed">` → the `embed` block, the inverse of `embedFigureHtml` and of V2's
+ * own `<figure data-rt="embed" data-url data-aspect><a href>title</a></figure>`. Only what is
+ * stored is read: the url (`data-url`, else the link's href), the title (`aria-label` when the
+ * figure has one, which ours always does, else the link's words, as V2 reads it) and the aspect
+ * (V2's four, else 16:9). The frame, the card and `data-provider` are derived and ignored.
+ */
+function embedFigureRule(key: KeyGenerator, schema: PtSchema | undefined): DeserializerRule {
+  const allowed = allowsBlockObject(schema, PT_TYPES.embed);
+  return {
+    deserialize(el: Node, _next, createBlock) {
+      if (!allowed || !isElement(el) || el.nodeName.toLowerCase() !== "figure" || el.getAttribute("data-rt") !== "embed") return undefined;
+      const link = el.querySelector("a[href]");
+      const url = (el.getAttribute("data-url") ?? link?.getAttribute("href") ?? "").trim();
+      const title = (el.hasAttribute("aria-label") ? el.getAttribute("aria-label") ?? "" : (link?.textContent ?? "")).trim().slice(0, EMBED_TITLE_MAX);
+      const aspect = el.getAttribute("data-aspect");
+      return createBlock({ _type: PT_TYPES.embed, _key: key(), url, title, aspect: isEmbedAspect(aspect) ? aspect : "16:9" });
+    },
+  };
+}
+
+/**
+ * RT2: V2's button, `<a data-button="true" href>`, → a `link` annotation with `button: true` (and
+ * the site's class, a new tab and `rel`, as the classed link keeps them). Gated on the schema's
+ * link annotation having the `button` field: a field without the button feature reads it as the
+ * plain link it also is.
+ */
+function buttonLinkRule(key: KeyGenerator, schema: PtSchema | undefined): DeserializerRule {
+  const allowed =
+    !schema || schema.annotations.some((a) => a.name === PT_TYPES.link && (a.fields ?? []).some((f) => f.name === "button"));
+  return {
+    deserialize(el: Node, next) {
+      if (!allowed || !isElement(el) || el.nodeName.toLowerCase() !== "a") return undefined;
+      const flag = el.getAttribute("data-button");
+      const href = el.getAttribute("href")?.trim();
+      if (flag === null || flag === "false" || !href) return undefined;
+      const className = el.getAttribute("class")?.trim();
+      const rel = el.getAttribute("rel")?.trim();
+      const markDef = {
+        _type: PT_TYPES.link,
+        _key: key(),
+        href,
+        button: true,
+        ...(className ? { className } : {}),
+        ...(el.getAttribute("target") === "_blank" ? { target: "_blank" } : {}),
+        ...(rel ? { rel } : {}),
+      };
+      return { _type: "__annotation", markDef, children: next(el.childNodes) } as never;
+    },
+  };
+}
+
 /**
  * `<a class>` → a `link` annotation that keeps the site's class as `className`.
  *
@@ -595,7 +783,7 @@ const PDF_HREF = /\.pdf(?:[?#]|$)/i;
  * and four rules would each have to ask that question. `undefined` hands the node back to the
  * deserialiser untouched, which is what keeps a mid-sentence PDF link a link.
  */
-function mediaRule(key: KeyGenerator, schema: PtSchema | undefined): DeserializerRule {
+function mediaRule(key: KeyGenerator, schema: PtSchema | undefined, onReceipt?: (line: string) => void): DeserializerRule {
   const allows = (name: string): boolean =>
     !schema || schema.blockObjects.some((o) => o.name === name);
   return {
@@ -628,6 +816,15 @@ function mediaRule(key: KeyGenerator, schema: PtSchema | undefined): Deserialize
           ...(caption ? { title: caption } : {}),
         });
       }
+      // RT3: a frame the embed allowlist recognises is an `embed`, in a field that takes one.
+      if (subject === "iframe" && allows(PT_TYPES.embed)) {
+        const src = (el.getAttribute("src") ?? "").trim();
+        if (src && resolveEmbed(src).kind === "frame") {
+          const title = ((el.getAttribute("title") ?? "").trim() || caption).slice(0, EMBED_TITLE_MAX);
+          const aspect = nearestEmbedAspect(el.getAttribute("width"), el.getAttribute("height"));
+          return createBlock({ _type: PT_TYPES.embed, _key: key(), url: src, title, aspect });
+        }
+      }
       // Image and embed keep the GENERIC block object under their existing `builtin:` keys —
       // thousands of stored values already spell them that way, and nothing is gained by a
       // migration when the renderer reads both shapes from the same fields.
@@ -636,6 +833,18 @@ function mediaRule(key: KeyGenerator, schema: PtSchema | undefined): Deserialize
       if (!src) return undefined;
       if (subject === "iframe") {
         const title = (el.getAttribute("title") ?? "").trim() || caption;
+        // Any other frame keeps the legacy, read-only `builtin:video` path: a working Calendly-like
+        // embed must not turn into a link (R15). Recorded, so the allowlist can grow from evidence.
+        // Only a frame the allowlist does not know is evidence; a known one here just met a field without the embed.
+        if (resolveEmbed(src).kind !== "frame") {
+          let host = "invalid";
+          try {
+            host = new URL(src).hostname;
+          } catch {
+            /* an unparsable src is recorded as such */
+          }
+          onReceipt?.(`embed.legacy_iframe:${host}`);
+        }
         return createBlock({
           _type: PT_TYPES.block,
           _key: key(),
@@ -795,11 +1004,39 @@ const allowsBlockObject = (schema: PtSchema | undefined, name: string): boolean
 const isElement = (n: unknown): n is Element =>
   !!n && typeof n === "object" && (n as Node).nodeType === 1;
 
+const BLOCK_TAGS = new Set([
+  "ADDRESS", "ARTICLE", "ASIDE", "BLOCKQUOTE", "DD", "DETAILS", "DIV", "DL", "DT", "FIGCAPTION", "FIGURE", "FOOTER",
+  "H1", "H2", "H3", "H4", "H5", "H6", "HEADER", "HR", "LI", "MAIN", "NAV", "OL", "P", "PRE", "SECTION", "TABLE", "UL",
+]);
+
+/**
+ * The parser, minus the indentation between block elements. The converter skips that whitespace
+ * at the top level but turns it into an empty block inside a wrapper, so `<div><p>a</p>\n<p>b</p></div>`
+ * came back as three blocks. Only ASCII whitespace beside a block element goes: a space between two
+ * inline elements is a word break, and `&nbsp;` or a `<br>` is an author's blank line.
+ */
+const withoutBlockWhitespace =
+  (parse: (html: string) => Document) =>
+  (html: string): Document => {
+    const doc = parse(html);
+    const walker = doc.createTreeWalker(doc.body ?? doc, 4 /* NodeFilter.SHOW_TEXT */);
+    const blank: Node[] = [];
+    for (let n = walker.nextNode(); n; n = walker.nextNode()) {
+      const beside = [n.previousSibling, n.nextSibling].some((s) => !!s && BLOCK_TAGS.has(s.nodeName));
+      if (beside && /^[ \t\n\r]*$/.test(n.nodeValue ?? "") && n.parentNode?.nodeName !== "PRE") blank.push(n);
+    }
+    for (const n of blank) n.parentNode?.removeChild(n);
+    return doc;
+  };
+
 export function htmlToPortableText(html: string, opts: IngestOptions = {}): PortableText {
   if (typeof html !== "string" || html.trim() === "") return [];
   const key = opts.keyGenerator ?? randomKey;
+  const parse =
+    opts.parseHtml ?? (typeof DOMParser !== "undefined" ? (markup: string) => new DOMParser().parseFromString(markup, "text/html") : undefined);
   const converted = ptHtmlToPortableText(html, {
-    ...(opts.parseHtml ? { parseHtml: opts.parseHtml } : {}),
+    // The browser's own parser when none is passed (Node must pass one), so both drop the same whitespace.
+    ...(parse ? { parseHtml: withoutBlockWhitespace(parse) } : {}),
     // `ptSchemaFor` returns a schema DEFINITION — the same object `EditorProvider` takes, so
     // the editor and this filter are literally configured from one value. The deserialiser
     // wants the COMPILED form, so the compile happens here rather than at every call site.
@@ -817,14 +1054,23 @@ export function htmlToPortableText(html: string, opts: IngestOptions = {}): Port
      * accepts only images.
      */
     rules: [
+      // First: a marker's content is the site's, never this field's text.
+      ...(opts.siteParts || allowsBlockObject(opts.schema, PT_TYPES.block) ? [siteRule(key)] : []),
       ...(allowsBlockObject(opts.schema, PT_TYPES.table)
         ? [tableFigureRule(key, (cell) => htmlToPortableText(cell, opts)), tableRule(key, (cell) => htmlToPortableText(cell, opts))]
         : []),
       ...(allowsBlockObject(opts.schema, PT_TYPES.code) ? [codeRule(key)] : []),
       ...(allowsBlockObject(opts.schema, PT_TYPES.pullQuote) ? [pullQuoteRule(key)] : []),
       ...(allowsBlockObject(opts.schema, PT_TYPES.keyTakeaways) ? [keyTakeawaysRule(key)] : []),
-      mediaRule(key, opts.schema),
+      // Before the media rule, which would take the iframe out of our own embed figure (and an image out of a gallery).
+      ...(allowsBlockObject(opts.schema, PT_TYPES.gallery) ? [galleryFigureRule(key)] : []),
+      ...(allowsBlockObject(opts.schema, PT_TYPES.tableOfContents) ? [tocRule(key)] : []),
+      embedFigureRule(key, opts.schema),
+      mediaRule(key, opts.schema, opts.onReceipt),
+      checklistItemRule(opts.schema),
+      ownDecoratorRule(opts.schema),
       ...(allowsStyled(opts.schema) ? [styledRule(key)] : []),
+      buttonLinkRule(key, opts.schema),
       ...(allowsLinks(opts.schema) ? [classedLinkRule(key)] : []),
     ],
     keyGenerator: key,

package/src/plain.ts

@@ -1,6 +1,7 @@
 import { isPortableTextBlock, isPortableTextSpan, PT_TYPES, type PortableText, type PortableTextTable } from "./types";
 import { migrateTable, tableGroupLayout } from "./table";
 import { keyTakeawaysView } from "./key-takeaways";
+import { galleryView } from "./gallery";
 import { componentCopyLines, detachedComponentCopy } from "./component-copy";
 
 /**
@@ -49,6 +50,11 @@ export function portableTextToPlainText(value: PortableText | null | undefined):
       lines.push(...(title ? [title] : []), ...points);
       continue;
     }
+    /** A gallery's captions are words a reader sees; its images are not. */
+    if (node._type === PT_TYPES.gallery) {
+      lines.push(...galleryView(node).images.map((x) => x.caption).filter(Boolean));
+      continue;
+    }
     const table = migrateTable(node);
     if (table) {
       lines.push(...tableLines(table));

package/src/read.ts

@@ -3,6 +3,7 @@ import { lexicalToPortableText } from "./from-lexical";
 import { migrateTable } from "./table";
 import { detachedComponentCopy, type DetachedComponentCopy } from "./component-copy";
 import { randomKey, type KeyGenerator } from "./key";
+import type { EmbedCard, RichTextEmbeds } from "@bettercms-ai/types";
 
 /**
  * THE reader. One function, every stored shape, and it never throws.
@@ -233,6 +234,31 @@ export function readStructuredContent(props: unknown): PortableText | null {
   return arr && arr.length > 0 ? (arr as PortableText) : null;
 }
 
+/**
+ * The embeds a delivered value carries beside its html (`RichTextEmbeds`), from the same envelope
+ * `readStructuredContent` reads; undefined when there are none. Delivered data is checked, not
+ * trusted: a card keeps only string fields.
+ */
+export function readStructuredEmbeds(props: unknown): RichTextEmbeds | undefined {
+  if (!isRecord(props)) return undefined;
+  const candidate = props.content ?? props.value;
+  const raw = isRecord(candidate) ? candidate.embeds : undefined;
+  if (!isRecord(raw)) return undefined;
+  const cards: Record<string, EmbedCard> = {};
+  if (isRecord(raw.cards)) {
+    for (const [url, c] of Object.entries(raw.cards)) {
+      if (!isRecord(c) || typeof c.text !== "string" || typeof c.authorName !== "string") continue;
+      cards[url] = {
+        text: c.text,
+        authorName: c.authorName,
+        ...(typeof c.authorUrl === "string" ? { authorUrl: c.authorUrl } : {}),
+        ...(typeof c.date === "string" ? { date: c.date } : {}),
+      };
+    }
+  }
+  return { cards };
+}
+
 /** The `component` block an embed in rich text stands for. */
 export interface EmbeddedComponentBlock {
   type: "component";