markdown-magic 2.6.1 → 3.0.0

package/lib/utils/html-to-json/lexer.js

@@ -0,0 +1,345 @@
+const {
+  startsWith,
+  endsWith,
+  stringIncludes,
+  arrayIncludes
+} = require('./compat')
+
+function feedPosition(position, str, len) {
+  const start = position.index
+  const end = position.index = start + len
+  for (let i = start; i < end; i++) {
+    const char = str.charAt(i)
+    if (char === '\n') {
+      position.line++
+      position.column = 0
+    } else {
+      position.column++
+    }
+  }
+}
+
+function jumpPosition (position, str, end) {
+  const len = end - position.index
+  return feedPosition(position, str, len)
+}
+
+function makeInitialPosition () {
+  return {
+    index: 0,
+    column: 0,
+    line: 1
+  }
+}
+
+function copyPositionStart(position) {
+  return {
+    index: position.index,
+    line: position.line,
+    column: position.column + 1
+  }
+}
+
+function copyPositionEnd(position) {
+  return {
+    index: position.index + 1,
+    line: position.line,
+    column: position.column + 1
+  }
+}
+
+// Old offset was index/column was 1 off in some cases
+function copyPosition(position) {
+  return {
+    index: position.index,
+    line: position.line,
+    column: position.column
+  }
+}
+
+function lexer (str, options) {
+  const state = {
+    str,
+    options,
+    position: makeInitialPosition(),
+    tokens: []
+  }
+  lex(state)
+  return state.tokens
+}
+
+function lex (state) {
+  const {str, options: {childlessTags}} = state
+  const len = str?.length || 0
+  while (state.position.index < len) {
+    const start = state.position.index
+    lexText(state)
+    if (state.position.index === start) {
+      const isComment = startsWith(str, '!--', start + 1)
+      if (isComment) {
+        lexComment(state)
+      } else {
+        const tagName = lexTag(state)
+        const safeTag = tagName.toLowerCase()
+        if (arrayIncludes(childlessTags, safeTag)) {
+          lexSkipTag(tagName, state)
+        }
+      }
+    }
+  }
+}
+
+const alphanumeric = /[A-Za-z0-9]/
+function findTextEnd (str, index) {
+  while (true) {
+    const textEnd = str.indexOf('<', index)
+    if (textEnd === -1) {
+      return textEnd
+    }
+    const char = str.charAt(textEnd + 1)
+    if (char === '/' || char === '!' || alphanumeric.test(char)) {
+      return textEnd
+    }
+    index = textEnd + 1
+  }
+}
+
+function lexText (state) {
+  const type = 'text'
+  const {str, position} = state
+  let textEnd = findTextEnd(str, position.index)
+  if (textEnd === position.index) return
+  if (textEnd === -1) {
+    textEnd = str.length
+  }
+
+  const start = copyPositionStart(position)
+  const content = str.slice(position.index, textEnd)
+  jumpPosition(position, str, textEnd)
+  const end = copyPositionEnd(position)
+  state.tokens.push({type, content, position: {start, end}})
+}
+
+function lexComment (state) {
+  const {str, position} = state
+  const start = copyPositionStart(position)
+  feedPosition(position, str, 4) // "<!--".length
+  let contentEnd = str.indexOf('-->', position.index)
+  let commentEnd = contentEnd + 3 // "-->".length
+  if (contentEnd === -1) {
+    contentEnd = commentEnd = str.length
+  }
+
+  const content = str.slice(position.index, contentEnd)
+  jumpPosition(position, str, commentEnd)
+  state.tokens.push({
+    type: 'comment',
+    content,
+    position: {
+      start,
+      end: copyPositionEnd(position)
+    }
+  })
+}
+
+function lexTag (state) {
+  const {str, position} = state
+  {
+    const secondChar = str.charAt(position.index + 1)
+    const close = secondChar === '/'
+    const start = copyPositionStart(position, 0)
+    feedPosition(position, str, close ? 2 : 1)
+    state.tokens.push({type: 'tag-start', close, position: {start}})
+  }
+  const tagName = lexTagName(state)
+  lexTagAttributes(state)
+  {
+    const firstChar = str.charAt(position.index)
+    const close = firstChar === '/'
+    feedPosition(position, str, close ? 2 : 1)
+    const end = copyPositionEnd(position)
+    state.tokens.push({type: 'tag-end', close, position: {end}})
+  }
+  return tagName
+}
+
+// See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#special-white-space
+const whitespace = /\s/
+function isWhitespaceChar (char) {
+  return whitespace.test(char)
+}
+
+function lexTagName (state) {
+  const {str, position} = state
+  const len = str.length
+  let start = position.index
+  while (start < len) {
+    const char = str.charAt(start)
+    const isTagChar = !(isWhitespaceChar(char) || char === '/' || char === '>')
+    if (isTagChar) break
+    start++
+  }
+
+  let end = start + 1
+  while (end < len) {
+    const char = str.charAt(end)
+    const isTagChar = !(isWhitespaceChar(char) || char === '/' || char === '>')
+    if (!isTagChar) break
+    end++
+  }
+
+  jumpPosition(position, str, end)
+  const tagName = str.slice(start, end)
+  state.tokens.push({
+    type: 'tag',
+    content: tagName
+  })
+  return tagName
+}
+
+function lexTagAttributes (state) {
+  const {str, position, tokens} = state
+  let cursor = position.index
+  let quote = null // null, single-, or double-quote
+  let wordBegin = cursor // index of word start
+  const words = [] // "key", "key=value", "key='value'", etc
+  const len = str.length
+  while (cursor < len) {
+    const char = str.charAt(cursor)
+    if (quote) {
+      const isQuoteEnd = char === quote
+      if (isQuoteEnd) {
+        quote = null
+      }
+      cursor++
+      continue
+    }
+
+    const isTagEnd = char === '/' || char === '>'
+    if (isTagEnd) {
+      if (cursor !== wordBegin) {
+        words.push(str.slice(wordBegin, cursor))
+      }
+      break
+    }
+
+    const isWordEnd = isWhitespaceChar(char)
+    if (isWordEnd) {
+      if (cursor !== wordBegin) {
+        words.push(str.slice(wordBegin, cursor))
+      }
+      wordBegin = cursor + 1
+      cursor++
+      continue
+    }
+
+    const isQuoteStart = char === '\'' || char === '"'
+    if (isQuoteStart) {
+      quote = char
+      cursor++
+      continue
+    }
+
+    cursor++
+  }
+  jumpPosition(position, str, cursor)
+
+  const wLen = words.length
+  const type = 'attribute'
+  for (let i = 0; i < wLen; i++) {
+    const word = words[i]
+    const isNotPair = word.indexOf('=') === -1
+    if (isNotPair) {
+      const secondWord = words[i + 1]
+      if (secondWord && startsWith(secondWord, '=')) {
+        if (secondWord.length > 1) {
+          const newWord = word + secondWord
+          tokens.push({type, content: newWord})
+          i += 1
+          continue
+        }
+        const thirdWord = words[i + 2]
+        i += 1
+        if (thirdWord) {
+          const newWord = word + '=' + thirdWord
+          tokens.push({type, content: newWord})
+          i += 1
+          continue
+        }
+      }
+    }
+    if (endsWith(word, '=')) {
+      const secondWord = words[i + 1]
+      if (secondWord && !stringIncludes(secondWord, '=')) {
+        const newWord = word + secondWord
+        tokens.push({type, content: newWord})
+        i += 1
+        continue
+      }
+
+      const newWord = word.slice(0, -1)
+      tokens.push({type, content: newWord})
+      continue
+    }
+
+    tokens.push({type, content: word})
+  }
+}
+
+const push = [].push
+
+function lexSkipTag (tagName, state) {
+  const {str, position, tokens} = state
+  const safeTagName = tagName.toLowerCase()
+  const len = str.length
+  let index = position.index
+  while (index < len) {
+    const nextTag = str.indexOf('</', index)
+    if (nextTag === -1) {
+      lexText(state)
+      break
+    }
+
+    const tagStartPosition = copyPositionStart(position)
+    jumpPosition(tagStartPosition, str, nextTag)
+    const tagState = {str, position: tagStartPosition, tokens: []}
+    const name = lexTag(tagState)
+    if (safeTagName !== name.toLowerCase()) {
+      index = tagState.position.index
+      continue
+    }
+
+    if (nextTag !== position.index) {
+      const textStart = copyPositionStart(position)
+      jumpPosition(position, str, nextTag)
+      tokens.push({
+        type: 'text',
+        content: str.slice(textStart.index, nextTag),
+        position: {
+          start: textStart,
+          end: copyPositionEnd(position)
+        }
+      })
+    }
+
+    push.apply(tokens, tagState.tokens)
+    jumpPosition(position, str, tagState.position.index)
+    break
+  }
+}
+
+module.exports = {
+  feedPosition,
+  jumpPosition,
+  makeInitialPosition,
+  copyPosition,
+  lexer,
+  lex,
+  findTextEnd,
+  lexText,
+  lexTag,
+  isWhitespaceChar,
+  lexTagName,
+  lexTagAttributes,
+  lexSkipTag,
+}

package/lib/utils/html-to-json/parser.js

@@ -0,0 +1,146 @@
+const { optionsParse } = require('../../options-parser')
+const { arrayIncludes }= require('./compat')
+
+function parser (tokens, options) {
+  const root = {tagName: null, children: []}
+  const state = {tokens, options, cursor: 0, stack: [root]}
+  parse(state)
+  return root.children
+}
+
+function hasTerminalParent (tagName, stack, terminals) {
+  const tagParents = terminals[tagName]
+  if (tagParents) {
+    let currentIndex = stack.length - 1
+    while (currentIndex >= 0) {
+      const parentTagName = stack[currentIndex].tagName
+      if (parentTagName === tagName) {
+        break
+      }
+      if (arrayIncludes(tagParents, parentTagName)) {
+        return true
+      }
+      currentIndex--
+    }
+  }
+  return false
+}
+
+function rewindStack (stack, newLength, childrenEndPosition, endPosition) {
+  stack[newLength].position.end = endPosition
+  for (let i = newLength + 1, len = stack.length; i < len; i++) {
+    stack[i].position.end = childrenEndPosition
+  }
+  stack.splice(newLength)
+}
+
+function parse (state) {
+  const {tokens, options} = state
+  let {stack} = state
+  let nodes = stack[stack.length - 1].children
+  const len = tokens.length
+  let {cursor} = state
+  while (cursor < len) {
+    const token = tokens[cursor]
+    if (token.type !== 'tag-start') {
+      nodes.push(token)
+      cursor++
+      continue
+    }
+
+    const tagToken = tokens[++cursor]
+    cursor++
+    const tagName = tagToken.content.toLowerCase()
+    if (token.close) {
+      let index = stack.length
+      let shouldRewind = false
+      while (--index > -1) {
+        if (stack[index].tagName === tagName) {
+          shouldRewind = true
+          break
+        }
+      }
+      while (cursor < len) {
+        const endToken = tokens[cursor]
+        if (endToken.type !== 'tag-end') break
+        cursor++
+      }
+      if (shouldRewind) {
+        rewindStack(stack, index, token.position.start, tokens[cursor - 1].position.end)
+        break
+      } else {
+        continue
+      }
+    }
+
+    const isClosingTag = arrayIncludes(options.closingTags, tagName)
+    let shouldRewindToAutoClose = isClosingTag
+    if (shouldRewindToAutoClose) {
+      const { closingTagAncestorBreakers: terminals } = options
+      shouldRewindToAutoClose = !hasTerminalParent(tagName, stack, terminals)
+    }
+
+    if (shouldRewindToAutoClose) {
+      // rewind the stack to just above the previous
+      // closing tag of the same name
+      let currentIndex = stack.length - 1
+      while (currentIndex > 0) {
+        if (tagName === stack[currentIndex].tagName) {
+          rewindStack(stack, currentIndex, token.position.start, token.position.start)
+          const previousIndex = currentIndex - 1
+          nodes = stack[previousIndex].children
+          break
+        }
+        currentIndex = currentIndex - 1
+      }
+    }
+
+    // let attributes = []
+    let propsRaw = ''
+    let attrToken
+    while (cursor < len) {
+      attrToken = tokens[cursor]
+      if (attrToken.type === 'tag-end') break
+      // attributes.push(attrToken.content)
+      propsRaw+= ((propsRaw !== '') ? ' ' : '') + attrToken.content
+      cursor++
+    }
+
+    cursor++
+    const children = []
+    const position = {
+      start: token.position.start,
+      end: attrToken.position.end
+    }
+    const elementNode = {
+      type: 'element',
+      tagName: tagToken.content,
+      // attributes,
+      props: (propsRaw) ? optionsParse(propsRaw) : {},
+      propsRaw,
+      children,
+      position
+    }
+    nodes.push(elementNode)
+
+    const hasChildren = !(attrToken.close || arrayIncludes(options.voidTags, tagName))
+    if (hasChildren) {
+      const size = stack.push({tagName, children, position})
+      const innerState = {tokens, options, cursor, stack}
+      parse(innerState)
+      cursor = innerState.cursor
+      const rewoundInElement = stack.length === size
+      if (rewoundInElement) {
+        elementNode.position.end = tokens[cursor - 1].position.end
+      }
+    }
+  }
+  state.cursor = cursor
+}
+
+module.exports = {
+  parser,
+  hasTerminalParent,
+  rewindStack,
+  parse,
+}

package/lib/utils/html-to-json/stringify.js

@@ -0,0 +1,37 @@
+const { arrayIncludes }= require('./compat')
+
+// Old
+function formatAttributes(attributes) {
+  return attributes.reduce((attrs, attribute) => {
+    const {key, value} = attribute
+    if (value === null) {
+      return `${attrs} ${key}`
+    }
+    const quoteEscape = value.indexOf('\'') !== -1
+    const quote = quoteEscape ? '"' : '\''
+    return `${attrs} ${key}=${quote}${value}${quote}`
+  }, '')
+}
+
+function toHTML(tree, options) {
+  return tree.map((node) => {
+    if (node.type === 'text') {
+      return node.content
+    }
+    if (node.type === 'comment') {
+      return `<!--${node.content}-->`
+    }
+    const {tagName, propsRaw, children} = node
+    // @TODO update prop parsing to keep new lines
+    const propsString = (propsRaw) ? ` ${propsRaw}` : ''
+    const isSelfClosing = arrayIncludes(options.voidTags, tagName.toLowerCase())
+    return isSelfClosing
+      ? `<${tagName}${propsString}>`
+      : `<${tagName}${propsString}>${toHTML(children, options)}</${tagName}>`
+  }).join('')
+}
+
+module.exports = {
+  formatAttributes,
+  toHTML
+}

package/lib/utils/html-to-json/tags.js

@@ -0,0 +1,171 @@
+/*
+  Tags which contain arbitrary non-parsed content
+  For example: <script> JavaScript should not be parsed
+*/
+const childlessTags = ['style', 'script', 'template']
+
+/*
+  Tags which auto-close because they cannot be nested
+  For example: <p>Outer<p>Inner is <p>Outer</p><p>Inner</p>
+*/
+const closingTags = [
+  'html', 'head', 'body', 'p', 'dt', 'dd', 'li', 'option',
+  'thead', 'th', 'tbody', 'tr', 'td', 'tfoot', 'colgroup'
+]
+
+/*
+  Closing tags which have ancestor tags which
+  may exist within them which prevent the
+  closing tag from auto-closing.
+  For example: in <li><ul><li></ul></li>,
+  the top-level <li> should not auto-close.
+*/
+const closingTagAncestorBreakers = {
+  li: ['ul', 'ol', 'menu'],
+  dt: ['dl'],
+  dd: ['dl'],
+  tbody: ['table'],
+  thead: ['table'],
+  tfoot: ['table'],
+  tr: ['table'],
+  td: ['table']
+}
+
+/*
+  Tags which do not need the closing tag
+  For example: <img> does not need </img>
+*/
+const voidTags = [
+  '!doctype', 'area', 'base', 'br', 'col', 'command',
+  'embed', 'hr', 'img', 'input', 'keygen', 'link',
+  'meta', 'param', 'source', 'track', 'wbr'
+]
+
+// https://github.com/sindresorhus/html-tags/blob/main/html-tags.json
+const htmlTags = [
+	"a",
+	"abbr",
+	"address",
+	"area",
+	"article",
+	"aside",
+	"audio",
+	"b",
+	"base",
+	"bdi",
+	"bdo",
+	"blockquote",
+	"body",
+	"br",
+	"button",
+	"canvas",
+	"caption",
+	"cite",
+	"code",
+	"col",
+	"colgroup",
+	"data",
+	"datalist",
+	"dd",
+	"del",
+	"details",
+	"dfn",
+	"dialog",
+	"div",
+	"dl",
+	"dt",
+	"em",
+	"embed",
+	"fieldset",
+	"figcaption",
+	"figure",
+	"footer",
+	"form",
+	"h1",
+	"h2",
+	"h3",
+	"h4",
+	"h5",
+	"h6",
+	"head",
+	"header",
+	"hgroup",
+	"hr",
+	"html",
+	"i",
+	"iframe",
+	"img",
+	"input",
+	"ins",
+	"kbd",
+	"label",
+	"legend",
+	"li",
+	"link",
+	"main",
+	"map",
+	"mark",
+	"math",
+	"menu",
+	"menuitem",
+	"meta",
+	"meter",
+	"nav",
+	"noscript",
+	"object",
+	"ol",
+	"optgroup",
+	"option",
+	"output",
+	"p",
+	"param",
+	"picture",
+	"pre",
+	"progress",
+	"q",
+	"rb",
+	"rp",
+	"rt",
+	"rtc",
+	"ruby",
+	"s",
+	"samp",
+	"script",
+	"section",
+	"select",
+	"slot",
+	"small",
+	"source",
+	"span",
+	"strong",
+	"style",
+	"sub",
+	"summary",
+	"sup",
+	"svg",
+	"table",
+	"tbody",
+	"td",
+	"template",
+	"textarea",
+	"tfoot",
+	"th",
+	"thead",
+	"time",
+	"title",
+	"tr",
+	"track",
+	"u",
+	"ul",
+	"var",
+	"video",
+	"wbr"
+]
+
+module.exports = {
+  childlessTags,
+  closingTags,
+  closingTagAncestorBreakers,
+  voidTags,
+  htmlTags
+}

package/lib/utils/index.js

@@ -0,0 +1,19 @@
+
+function onlyUnique(value, index, self) {
+  return self.indexOf(value) === index
+}
+
+function getCodeLocation(srcPath, line, column = '0') {
+  return `${srcPath}:${line}:${column}`
+}
+
+function pluralize(thing, single = '', plural = '') {
+  const count = Array.isArray(thing) ? thing.length : Number(thing)
+  return count === 1 ? single : plural
+}
+
+module.exports = {
+  pluralize,
+  onlyUnique,
+  getCodeLocation
+}