@portabletext/block-tools 0.0.0

package/src/HtmlDeserializer/index.ts

@@ -0,0 +1,313 @@
+import type {
+  ArraySchemaType,
+  PortableTextBlock,
+  PortableTextObject,
+  PortableTextTextBlock,
+} from '@sanity/types'
+import {flatten} from 'lodash'
+import type {
+  ArbitraryTypedObject,
+  DeserializerRule,
+  HtmlDeserializerOptions,
+  PlaceholderAnnotation,
+  PlaceholderDecorator,
+  TypedObject,
+} from '../types'
+import {findBlockType} from '../util/findBlockType'
+import {resolveJsType} from '../util/resolveJsType'
+import {
+  createRuleOptions,
+  defaultParseHtml,
+  ensureRootIsBlocks,
+  flattenNestedBlocks,
+  isMinimalBlock,
+  isMinimalSpan,
+  isNodeList,
+  isPlaceholderAnnotation,
+  isPlaceholderDecorator,
+  preprocess,
+  tagName,
+  trimWhitespace,
+} from './helpers'
+import {createRules} from './rules'
+
+/**
+ * HTML Deserializer
+ *
+ */
+export default class HtmlDeserializer {
+  blockContentType: ArraySchemaType
+  rules: DeserializerRule[]
+  parseHtml: (html: string) => HTMLElement
+  _markDefs: PortableTextObject[] = []
+
+  /**
+   * Create a new serializer respecting a Sanity block content type's schema
+   *
+   * @param blockContentType - Schema type for array containing _at least_ a block child type
+   * @param options - Options for the deserialization process
+   */
+  constructor(
+    blockContentType: ArraySchemaType,
+    options: HtmlDeserializerOptions = {},
+  ) {
+    const {rules = [], unstable_whitespaceOnPasteMode = 'preserve'} = options
+    if (!blockContentType) {
+      throw new Error("Parameter 'blockContentType' is required")
+    }
+    const standardRules = createRules(
+      blockContentType,
+      createRuleOptions(blockContentType),
+    )
+    this.rules = [...rules, ...standardRules]
+    const parseHtml = options.parseHtml || defaultParseHtml()
+    this.blockContentType = blockContentType
+    this.parseHtml = (html) => {
+      const doc = preprocess(html, parseHtml, {unstable_whitespaceOnPasteMode})
+      return doc.body
+    }
+  }
+
+  /**
+   * Deserialize HTML.
+   *
+   * @param html - The HTML to deserialize, as a string
+   * @returns Array of blocks - either portable text blocks or other allowed blocks
+   */
+  deserialize = (html: string): TypedObject[] => {
+    this._markDefs = []
+    const {parseHtml} = this
+    const fragment = parseHtml(html)
+    const children = Array.from(fragment.childNodes) as HTMLElement[]
+    // Ensure that there are no blocks within blocks, and trim whitespace
+    const blocks = trimWhitespace(
+      flattenNestedBlocks(
+        ensureRootIsBlocks(this.deserializeElements(children)),
+      ),
+    )
+
+    if (this._markDefs.length > 0) {
+      blocks
+        .filter(
+          (block): block is PortableTextTextBlock => block._type === 'block',
+        )
+        .forEach((block) => {
+          block.markDefs = block.markDefs || []
+          block.markDefs = block.markDefs.concat(
+            this._markDefs.filter((def) => {
+              return flatten(
+                block.children.map((child) => child.marks || []),
+              ).includes(def._key)
+            }),
+          )
+        })
+    }
+
+    // Set back the potentially hoisted block type
+    const type = this.blockContentType.of.find(findBlockType)
+    if (!type) {
+      return blocks
+    }
+
+    return blocks.map((block) => {
+      if (block._type === 'block') {
+        block._type = type.name
+      }
+      return block
+    })
+  }
+
+  /**
+   * Deserialize an array of DOM elements.
+   *
+   * @param elements - Array of DOM elements to deserialize
+   * @returns
+   */
+  deserializeElements = (elements: Node[] = []): TypedObject[] => {
+    let nodes: TypedObject[] = []
+    elements.forEach((element) => {
+      nodes = nodes.concat(this.deserializeElement(element))
+    })
+    return nodes
+  }
+
+  /**
+   * Deserialize a DOM element
+   *
+   * @param element - Deserialize a DOM element
+   * @returns
+   */
+  deserializeElement = (element: Node): TypedObject | TypedObject[] => {
+    const next = (
+      elements: Node | Node[] | NodeList,
+    ): TypedObject | TypedObject[] | undefined => {
+      if (isNodeList(elements)) {
+        return this.deserializeElements(Array.from(elements))
+      }
+
+      if (Array.isArray(elements)) {
+        return this.deserializeElements(elements)
+      }
+
+      if (!elements) {
+        return undefined
+      }
+
+      return this.deserializeElement(elements)
+    }
+
+    const block = (props: ArbitraryTypedObject) => {
+      return {
+        _type: '__block',
+        block: props,
+      }
+    }
+
+    let node: TypedObject | Array<TypedObject> | undefined
+    for (let i = 0; i < this.rules.length; i++) {
+      const rule = this.rules[i]
+      if (!rule.deserialize) {
+        continue
+      }
+
+      const ret = rule.deserialize(element, next, block)
+      const type = resolveJsType(ret)
+
+      if (
+        type !== 'array' &&
+        type !== 'object' &&
+        type !== 'null' &&
+        type !== 'undefined'
+      ) {
+        throw new Error(
+          `A rule returned an invalid deserialized representation: "${node}".`,
+        )
+      }
+
+      if (ret === undefined) {
+        continue
+      } else if (ret === null) {
+        throw new Error('Deserializer rule returned `null`')
+      } else if (Array.isArray(ret)) {
+        node = ret
+      } else if (isPlaceholderDecorator(ret)) {
+        node = this.deserializeDecorator(ret)
+      } else if (isPlaceholderAnnotation(ret)) {
+        node = this.deserializeAnnotation(ret)
+      } else {
+        node = ret
+      }
+
+      // Set list level on list item
+      if (
+        ret &&
+        !Array.isArray(ret) &&
+        isMinimalBlock(ret) &&
+        'listItem' in ret
+      ) {
+        let parent = element.parentNode?.parentNode
+        while (parent && tagName(parent) === 'li') {
+          parent = parent.parentNode?.parentNode
+          ret.level = ret.level ? ret.level + 1 : 1
+        }
+      }
+
+      // Set newlines on spans orginating from a block element within a blockquote
+      if (
+        ret &&
+        !Array.isArray(ret) &&
+        isMinimalBlock(ret) &&
+        ret.style === 'blockquote'
+      ) {
+        ret.children.forEach((child, index) => {
+          if (isMinimalSpan(child) && child.text === '\r') {
+            child.text = '\n\n'
+            if (index === 0 || index === ret.children.length - 1) {
+              ret.children.splice(index, 1)
+            }
+          }
+        })
+      }
+      break
+    }
+
+    return node || next(element.childNodes) || []
+  }
+
+  /**
+   * Deserialize a `__decorator` type
+   * (an internal made up type to process decorators exclusively)
+   *
+   * @param decorator -
+   * @returns array of ...
+   */
+  deserializeDecorator = (decorator: PlaceholderDecorator): TypedObject[] => {
+    const {name} = decorator
+    const applyDecorator = (node: TypedObject) => {
+      if (isPlaceholderDecorator(node)) {
+        return this.deserializeDecorator(node)
+      } else if (isMinimalSpan(node)) {
+        node.marks = node.marks || []
+        if (node.text.trim()) {
+          // Only apply marks if this is an actual text
+          node.marks.unshift(name)
+        }
+      } else if (
+        'children' in node &&
+        Array.isArray((node as PortableTextBlock).children)
+      ) {
+        const block = node as any
+        block.children = block.children.map(applyDecorator)
+      }
+      return node
+    }
+    return decorator.children.reduce((children, node) => {
+      const ret = applyDecorator(node)
+      if (Array.isArray(ret)) {
+        return children.concat(ret)
+      }
+      children.push(ret)
+      return children
+    }, [] as TypedObject[])
+  }
+
+  /**
+   * Deserialize a `__annotation` object.
+   * (an internal made up type to process annotations exclusively)
+   *
+   * @param annotation -
+   * @returns Array of...
+   */
+  deserializeAnnotation = (
+    annotation: PlaceholderAnnotation,
+  ): TypedObject[] => {
+    const {markDef} = annotation
+    this._markDefs.push(markDef)
+    const applyAnnotation = (node: TypedObject) => {
+      if (isPlaceholderAnnotation(node)) {
+        return this.deserializeAnnotation(node)
+      } else if (isMinimalSpan(node)) {
+        node.marks = node.marks || []
+        if (node.text.trim()) {
+          // Only apply marks if this is an actual text
+          node.marks.unshift(markDef._key)
+        }
+      } else if (
+        'children' in node &&
+        Array.isArray((node as PortableTextBlock).children)
+      ) {
+        const block = node as any
+        block.children = block.children.map(applyAnnotation)
+      }
+      return node
+    }
+    return annotation.children.reduce((children, node) => {
+      const ret = applyAnnotation(node)
+      if (Array.isArray(ret)) {
+        return children.concat(ret)
+      }
+      children.push(ret)
+      return children
+    }, [] as TypedObject[])
+  }
+}

package/src/HtmlDeserializer/preprocessors/gdocs.ts

@@ -0,0 +1,86 @@
+import type {HtmlPreprocessorOptions} from '../../types'
+import {normalizeWhitespace, removeAllWhitespace, tagName} from '../helpers'
+import {_XPathResult} from './xpathResult'
+
+export default (
+  _html: string,
+  doc: Document,
+  options: HtmlPreprocessorOptions,
+): Document => {
+  const whitespaceOnPasteMode =
+    options?.unstable_whitespaceOnPasteMode || 'preserve'
+  let gDocsRootOrSiblingNode = doc
+    .evaluate(
+      '//*[@id and contains(@id, "docs-internal-guid")]',
+      doc,
+      null,
+      _XPathResult.ORDERED_NODE_ITERATOR_TYPE,
+      null,
+    )
+    .iterateNext()
+
+  if (gDocsRootOrSiblingNode) {
+    const isWrappedRootTag = tagName(gDocsRootOrSiblingNode) === 'b'
+
+    // If this document isn't wrapped in a 'b' tag, then assume all siblings live on the root level
+    if (!isWrappedRootTag) {
+      gDocsRootOrSiblingNode = doc.body
+    }
+
+    switch (whitespaceOnPasteMode) {
+      case 'normalize':
+        // Keep only 1 empty block between content nodes
+        normalizeWhitespace(gDocsRootOrSiblingNode)
+        break
+      case 'remove':
+        // Remove all whitespace nodes
+        removeAllWhitespace(gDocsRootOrSiblingNode)
+        break
+      default:
+        break
+    }
+
+    // Tag every child with attribute 'is-google-docs' so that the GDocs rule-set can
+    // work exclusivly on these children
+    const childNodes = doc.evaluate(
+      '//*',
+      doc,
+      null,
+      _XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE,
+      null,
+    )
+
+    for (let i = childNodes.snapshotLength - 1; i >= 0; i--) {
+      const elm = childNodes.snapshotItem(i) as HTMLElement
+      elm?.setAttribute('data-is-google-docs', 'true')
+
+      if (
+        elm?.parentElement === gDocsRootOrSiblingNode ||
+        (!isWrappedRootTag && elm.parentElement === doc.body)
+      ) {
+        elm?.setAttribute('data-is-root-node', 'true')
+        tagName(elm)
+      }
+
+      // Handle checkmark lists - The first child of a list item is an image with a checkmark, and the serializer
+      // expects the first child to be the text node
+      if (
+        tagName(elm) === 'li' &&
+        elm.firstChild &&
+        tagName(elm?.firstChild) === 'img'
+      ) {
+        elm.removeChild(elm.firstChild)
+      }
+    }
+
+    // Remove that 'b' which Google Docs wraps the HTML content in
+    if (isWrappedRootTag) {
+      doc.body.firstElementChild?.replaceWith(
+        ...Array.from(gDocsRootOrSiblingNode.childNodes),
+      )
+    }
+
+    return doc
+  }
+  return doc
+}

package/src/HtmlDeserializer/preprocessors/html.ts

@@ -0,0 +1,57 @@
+import {_XPathResult} from './xpathResult'
+
+// Remove this cruft from the document
+const unwantedWordDocumentPaths = [
+  '/html/text()',
+  '/html/head/text()',
+  '/html/body/text()',
+  '/html/body/ul/text()',
+  '/html/body/ol/text()',
+  '//comment()',
+  '//style',
+  '//xml',
+  '//script',
+  '//meta',
+  '//link',
+]
+
+export default (_html: string, doc: Document): Document => {
+  // Make sure text directly on the body is wrapped in spans.
+  // This mimics what the browser does before putting html on the clipboard,
+  // when used in a script context with JSDOM
+  const bodyTextNodes = doc.evaluate(
+    '/html/body/text()',
+    doc,
+    null,
+    _XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE,
+    null,
+  )
+
+  for (let i = bodyTextNodes.snapshotLength - 1; i >= 0; i--) {
+    const node = bodyTextNodes.snapshotItem(i) as HTMLElement
+    const text = node.textContent || ''
+    if (text.replace(/[^\S\n]+$/g, '')) {
+      const newNode = doc.createElement('span')
+      newNode.appendChild(doc.createTextNode(text))
+      node.parentNode?.replaceChild(newNode, node)
+    } else {
+      node.parentNode?.removeChild(node)
+    }
+  }
+
+  const unwantedNodes = doc.evaluate(
+    unwantedWordDocumentPaths.join('|'),
+    doc,
+    null,
+    _XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE,
+    null,
+  )
+  for (let i = unwantedNodes.snapshotLength - 1; i >= 0; i--) {
+    const unwanted = unwantedNodes.snapshotItem(i)
+    if (!unwanted) {
+      continue
+    }
+    unwanted.parentNode?.removeChild(unwanted)
+  }
+  return doc
+}

package/src/HtmlDeserializer/preprocessors/index.ts

@@ -0,0 +1,13 @@
+import preprocessGDocs from './gdocs'
+import preprocessHTML from './html'
+import preprocessNotion from './notion'
+import preprocessWhitespace from './whitespace'
+import preprocessWord from './word'
+
+export default [
+  preprocessWhitespace,
+  preprocessNotion,
+  preprocessWord,
+  preprocessGDocs,
+  preprocessHTML,
+]

package/src/HtmlDeserializer/preprocessors/notion.ts

@@ -0,0 +1,25 @@
+import {_XPathResult} from './xpathResult'
+
+export default (html: string, doc: Document): Document => {
+  const NOTION_REGEX = /<!-- notionvc:.*?-->/g
+
+  if (html.match(NOTION_REGEX)) {
+    // Tag every child with attribute 'is-notion' so that the Notion rule-set can
+    // work exclusivly on these children
+    const childNodes = doc.evaluate(
+      '//*',
+      doc,
+      null,
+      _XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE,
+      null,
+    )
+
+    for (let i = childNodes.snapshotLength - 1; i >= 0; i--) {
+      const elm = childNodes.snapshotItem(i) as HTMLElement
+      elm?.setAttribute('data-is-notion', 'true')
+    }
+
+    return doc
+  }
+  return doc
+}

package/src/HtmlDeserializer/preprocessors/whitespace.ts

@@ -0,0 +1,31 @@
+import {PRESERVE_WHITESPACE_TAGS} from '../../constants'
+import {_XPathResult} from './xpathResult'
+
+export default (_: string, doc: Document): Document => {
+  // Recursively process all nodes.
+  function processNode(node: Node) {
+    // If this is a text node and not inside a tag where whitespace should be preserved, process it.
+    if (
+      node.nodeType === _XPathResult.BOOLEAN_TYPE &&
+      !PRESERVE_WHITESPACE_TAGS.includes(
+        node.parentElement?.tagName.toLowerCase() || '',
+      )
+    ) {
+      node.textContent =
+        node.textContent
+          ?.replace(/\s\s+/g, ' ') // Remove multiple whitespace
+          .replace(/[\r\n]+/g, ' ') || '' // Replace newlines with spaces
+    }
+    // Otherwise, if this node has children, process them.
+    else {
+      for (let i = 0; i < node.childNodes.length; i++) {
+        processNode(node.childNodes[i])
+      }
+    }
+  }
+
+  // Process all nodes starting from the root.
+  processNode(doc.body)
+
+  return doc
+}

package/src/HtmlDeserializer/preprocessors/word.ts

@@ -0,0 +1,92 @@
+import {_XPathResult} from './xpathResult'
+
+const WORD_HTML_REGEX =
+  /(class="?Mso|style=(?:"|')[^"]*?\bmso-|w:WordDocument|<o:\w+>|<\/font>)/
+
+// xPaths for elements that will be removed from the document
+const unwantedPaths = [
+  '//o:p',
+  "//span[@style='mso-list:Ignore']",
+  "//span[@style='mso-list: Ignore']",
+]
+
+// xPaths for elements that needs to be remapped into other tags
+const mappedPaths = [
+  "//p[@class='MsoTocHeading']",
+  "//p[@class='MsoTitle']",
+  "//p[@class='MsoToaHeading']",
+  "//p[@class='MsoSubtitle']",
+  "//span[@class='MsoSubtleEmphasis']",
+  "//span[@class='MsoIntenseEmphasis']",
+]
+
+// Which HTML element(s) to map the elements matching mappedPaths into
+const elementMap: Record<string, string[] | undefined> = {
+  MsoTocHeading: ['h3'],
+  MsoTitle: ['h1'],
+  MsoToaHeading: ['h2'],
+  MsoSubtitle: ['h5'],
+  MsoSubtleEmphasis: ['span', 'em'],
+  MsoIntenseEmphasis: ['span', 'em', 'strong'],
+  // Remove cruft
+}
+
+function isWordHtml(html: string) {
+  return WORD_HTML_REGEX.test(html)
+}
+
+export default (html: string, doc: Document): Document => {
+  if (!isWordHtml(html)) {
+    return doc
+  }
+
+  const unwantedNodes = doc.evaluate(
+    unwantedPaths.join('|'),
+    doc,
+    (prefix) => {
+      if (prefix === 'o') {
+        return 'urn:schemas-microsoft-com:office:office'
+      }
+      return null
+    },
+    _XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE,
+    null,
+  )
+
+  for (let i = unwantedNodes.snapshotLength - 1; i >= 0; i--) {
+    const unwanted = unwantedNodes.snapshotItem(i)
+    if (unwanted?.parentNode) {
+      unwanted.parentNode.removeChild(unwanted)
+    }
+  }
+
+  // Transform mapped elements into what they should be mapped to
+  const mappedElements = doc.evaluate(
+    mappedPaths.join('|'),
+    doc,
+    null,
+    _XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE,
+    null,
+  )
+  for (let i = mappedElements.snapshotLength - 1; i >= 0; i--) {
+    const mappedElm = mappedElements.snapshotItem(i) as HTMLElement
+    const tags = elementMap[mappedElm.className]
+    const text = doc.createTextNode(mappedElm.textContent || '')
+    if (!tags) {
+      continue
+    }
+
+    const parentElement = doc.createElement(tags[0])
+    let parent = parentElement
+    let child = parentElement
+    tags.slice(1).forEach((tag) => {
+      child = doc.createElement(tag)
+      parent.appendChild(child)
+      parent = child
+    })
+    child.appendChild(text)
+    mappedElm?.parentNode?.replaceChild(parentElement, mappedElm)
+  }
+
+  return doc
+}

package/src/HtmlDeserializer/preprocessors/xpathResult.ts

@@ -0,0 +1,13 @@
+// We need this here if run server side
+export const _XPathResult = {
+  ANY_TYPE: 0,
+  NUMBER_TYPE: 1,
+  STRING_TYPE: 2,
+  BOOLEAN_TYPE: 3,
+  UNORDERED_NODE_ITERATOR_TYPE: 4,
+  ORDERED_NODE_ITERATOR_TYPE: 5,
+  UNORDERED_NODE_SNAPSHOT_TYPE: 6,
+  ORDERED_NODE_SNAPSHOT_TYPE: 7,
+  ANY_UNORDERED_NODE_TYPE: 8,
+  FIRST_ORDERED_NODE_TYPE: 9,
+}