@yozora/tokenizer-definition 2.1.3 → 2.1.5

package/src/util/link-destination.ts

@@ -1,160 +0,0 @@
-import type { INodePoint } from '@yozora/character'
-import {
-  AsciiCodePoint,
-  VirtualCodePoint,
-  isAsciiControlCharacter,
-  isWhitespaceCharacter,
-} from '@yozora/character'
-import { eatOptionalWhitespaces } from '@yozora/core-tokenizer'
-
-/**
- * The processing token of eatAndCollectLinkDestination, used to save
- * intermediate data to support multiple codePosition fragment processing
- *
- * @see https://github.github.com/gfm/#link-destination
- */
-export interface ILinkDestinationCollectingState {
-  /**
-   * Whether the current token has collected a legal LinkDestination
-   */
-  saturated: boolean
-  /**
-   * Collected token points
-   */
-  nodePoints: INodePoint[]
-  /**
-   * Whether an opening angle bracket has been matched
-   */
-  hasOpenAngleBracket: boolean
-  /**
-   * Number of parentheses encountered
-   */
-  openParensCount: number
-}
-
-/**
- *
- * @param nodePoints
- * @param startIndex
- * @param endIndex
- * @param state
- * @see https://github.github.com/gfm/#link-destination
- */
-export function eatAndCollectLinkDestination(
-  nodePoints: ReadonlyArray<INodePoint>,
-  startIndex: number,
-  endIndex: number,
-  state: ILinkDestinationCollectingState | null,
-): { nextIndex: number; state: ILinkDestinationCollectingState } {
-  let i = startIndex
-
-  // init token
-  if (state == null) {
-    // eslint-disable-next-line no-param-reassign
-    state = {
-      saturated: false,
-      nodePoints: [],
-      hasOpenAngleBracket: false,
-      openParensCount: 0,
-    }
-  }
-
-  /**
-   * Although link destination may span multiple lines,
-   * they may not contain a blank line.
-   */
-  const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex)
-  if (firstNonWhitespaceIndex >= endIndex) return { nextIndex: -1, state: state }
-
-  if (state.nodePoints.length <= 0) {
-    i = firstNonWhitespaceIndex
-
-    // check whether in pointy brackets
-    const p = nodePoints[i]
-    if (p.codePoint === AsciiCodePoint.OPEN_ANGLE) {
-      i += 1
-      // eslint-disable-next-line no-param-reassign
-      state.hasOpenAngleBracket = true
-      state.nodePoints.push(p)
-    }
-  }
-
-  /**
-   * In pointy brackets:
-   *  - A sequence of zero or more characters between an opening '<' and
-   *    a closing '>' that contains no line breaks or unescaped '<' or '>' characters
-   */
-  if (state.hasOpenAngleBracket) {
-    for (; i < endIndex; ++i) {
-      const p = nodePoints[i]
-      switch (p.codePoint) {
-        case AsciiCodePoint.BACKSLASH:
-          if (i + 1 < endIndex) {
-            state.nodePoints.push(p)
-            state.nodePoints.push(nodePoints[i + 1])
-          }
-          i += 1
-          break
-        case AsciiCodePoint.OPEN_ANGLE:
-        case VirtualCodePoint.LINE_END:
-          return { nextIndex: -1, state: state }
-        case AsciiCodePoint.CLOSE_ANGLE:
-          // eslint-disable-next-line no-param-reassign
-          state.saturated = true
-          state.nodePoints.push(p)
-          return { nextIndex: i + 1, state: state }
-        default:
-          state.nodePoints.push(p)
-      }
-    }
-    return { nextIndex: i, state: state }
-  }
-
-  /**
-   * Not in pointy brackets:
-   *  - A nonempty sequence of characters that does not start with '<', does not include
-   *    ASCII space or control characters, and includes parentheses only if
-   *
-   *    a) they are backslash-escaped or
-   *    b) they are part of a balanced pair of unescaped parentheses. (Implementations
-   *       may impose limits on parentheses nesting to avoid performance issues,
-   *       but at least three levels of nesting should be supported.)
-   */
-  for (; i < endIndex; ++i) {
-    const p = nodePoints[i]
-    switch (p.codePoint) {
-      case AsciiCodePoint.BACKSLASH:
-        if (i + 1 < endIndex) {
-          state.nodePoints.push(p)
-          state.nodePoints.push(nodePoints[i + 1])
-        }
-        i += 1
-        break
-      case AsciiCodePoint.OPEN_PARENTHESIS:
-        // eslint-disable-next-line no-param-reassign
-        state.openParensCount += 1
-        state.nodePoints.push(p)
-        break
-      case AsciiCodePoint.CLOSE_PARENTHESIS:
-        // eslint-disable-next-line no-param-reassign
-        state.openParensCount -= 1
-        state.nodePoints.push(p)
-        if (state.openParensCount < 0) {
-          return { nextIndex: i, state: state }
-        }
-        break
-      default:
-        if (isWhitespaceCharacter(p.codePoint) || isAsciiControlCharacter(p.codePoint)) {
-          // eslint-disable-next-line no-param-reassign
-          state.saturated = true
-          return { nextIndex: i, state: state }
-        }
-        state.nodePoints.push(p)
-        break
-    }
-  }
-
-  // eslint-disable-next-line no-param-reassign
-  state.saturated = true
-  return { nextIndex: i, state: state }
-}

package/src/util/link-label.ts

@@ -1,116 +0,0 @@
-import type { INodePoint } from '@yozora/character'
-import { AsciiCodePoint, isWhitespaceCharacter } from '@yozora/character'
-import { eatOptionalWhitespaces } from '@yozora/core-tokenizer'
-
-/**
- * The processing token of eatAndCollectLinkLabel, used to save
- * intermediate data to support multiple codePosition fragment processing
- *
- * @see https://github.github.com/gfm/#link-label
- */
-export interface ILinkLabelCollectingState {
-  /**
-   * Whether the current token has collected a legal LinkDestination
-   */
-  saturated: boolean
-  /**
-   * Collected token points
-   */
-  nodePoints: INodePoint[]
-  /**
-   * Does it contain non-blank characters
-   */
-  hasNonWhitespaceCharacter: boolean
-}
-
-/**
- * A link label begins with a left bracket '[' and ends with the first right bracket ']'
- * that is not backslash-escaped. Between these brackets there must be at least one
- * non-whitespace character. Unescaped square bracket characters are not allowed inside
- * the opening and closing square brackets of link labels. A link label can have at most
- * 999 characters inside the square brackets.
- *
- * One label matches another just in case their normalized forms are equal. To normalize
- * a label, strip off the opening and closing brackets, perform the Unicode case fold,
- * strip leading and trailing whitespace and collapse consecutive internal whitespace to
- * a single space. If there are multiple matching reference link definitions, the one that
- * comes first in the document is used. (It is desirable in such cases to emit a warning.)
- *
- * @param nodePoints
- * @param startIndex
- * @param endIndex
- * @param state
- * @see https://github.github.com/gfm/#link-label
- */
-export function eatAndCollectLinkLabel(
-  nodePoints: ReadonlyArray<INodePoint>,
-  startIndex: number,
-  endIndex: number,
-  state: ILinkLabelCollectingState | null,
-): { nextIndex: number; state: ILinkLabelCollectingState } {
-  let i = startIndex
-
-  // init token
-  if (state == null) {
-    // eslint-disable-next-line no-param-reassign
-    state = {
-      saturated: false,
-      nodePoints: [],
-      hasNonWhitespaceCharacter: false,
-    }
-  }
-
-  /**
-   * Although link label may span multiple lines,
-   * they may not contain a blank line.
-   */
-  const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex)
-  if (firstNonWhitespaceIndex >= endIndex) return { nextIndex: -1, state: state }
-
-  if (state.nodePoints.length <= 0) {
-    i = firstNonWhitespaceIndex
-
-    // check whether in brackets
-    const p = nodePoints[i]
-    if (p.codePoint !== AsciiCodePoint.OPEN_BRACKET) {
-      return { nextIndex: -1, state: state }
-    }
-
-    i += 1
-    // eslint-disable-next-line no-param-reassign
-    state.nodePoints.push(p)
-  }
-
-  for (; i < endIndex; ++i) {
-    const p = nodePoints[i]
-    switch (p.codePoint) {
-      case AsciiCodePoint.BACKSLASH:
-        // eslint-disable-next-line no-param-reassign
-        state.hasNonWhitespaceCharacter = true
-        if (i + 1 < endIndex) {
-          state.nodePoints.push(p)
-          state.nodePoints.push(nodePoints[i + 1])
-        }
-        i += 1
-        break
-      case AsciiCodePoint.OPEN_BRACKET:
-        return { nextIndex: -1, state: state }
-      case AsciiCodePoint.CLOSE_BRACKET:
-        state.nodePoints.push(p)
-        if (state.hasNonWhitespaceCharacter) {
-          // eslint-disable-next-line no-param-reassign
-          state.saturated = true
-          return { nextIndex: i + 1, state: state }
-        }
-        return { nextIndex: -1, state: state }
-      default:
-        if (!isWhitespaceCharacter(p.codePoint)) {
-          // eslint-disable-next-line no-param-reassign
-          state.hasNonWhitespaceCharacter = true
-        }
-        state.nodePoints.push(p)
-    }
-  }
-
-  return { nextIndex: 1, state: state }
-}

package/src/util/link-title.ts

@@ -1,143 +0,0 @@
-import type { INodePoint } from '@yozora/character'
-import { AsciiCodePoint, VirtualCodePoint } from '@yozora/character'
-import { eatOptionalWhitespaces } from '@yozora/core-tokenizer'
-
-/**
- * The processing token of eatAndCollectLinkDestination, used to save
- * intermediate data to support multiple codePosition fragment processing.
- *
- * @see https://github.github.com/gfm/#link-title
- */
-export interface ILinkTitleCollectingState {
-  /**
-   * Whether the current token has collected a legal LinkDestination
-   */
-  saturated: boolean
-  /**
-   * Collected token points
-   */
-  nodePoints: INodePoint[]
-  /**
-   * Character that wrap link-title
-   */
-  wrapSymbol: number | null
-}
-
-/**
- *
- * @param nodePoints
- * @param startIndex
- * @param endIndex
- * @param state
- * @see https://github.github.com/gfm/#link-title
- */
-export function eatAndCollectLinkTitle(
-  nodePoints: ReadonlyArray<INodePoint>,
-  startIndex: number,
-  endIndex: number,
-  state: ILinkTitleCollectingState | null,
-): { nextIndex: number; state: ILinkTitleCollectingState } {
-  let i = startIndex
-
-  // init token
-  if (state == null) {
-    // eslint-disable-next-line no-param-reassign
-    state = {
-      saturated: false,
-      nodePoints: [],
-      wrapSymbol: null,
-    }
-  }
-
-  /**
-   * Although link titles may span multiple lines,
-   * they may not contain a blank line.
-   */
-  const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex)
-  if (firstNonWhitespaceIndex >= endIndex) return { nextIndex: -1, state: state }
-
-  if (state.nodePoints.length <= 0) {
-    i = firstNonWhitespaceIndex
-    const p = nodePoints[i]
-
-    switch (p.codePoint) {
-      case AsciiCodePoint.DOUBLE_QUOTE:
-      case AsciiCodePoint.SINGLE_QUOTE:
-      case AsciiCodePoint.OPEN_PARENTHESIS:
-        // eslint-disable-next-line no-param-reassign
-        state.wrapSymbol = p.codePoint
-        state.nodePoints.push(p)
-        i += 1
-        break
-      default:
-        return { nextIndex: -1, state: state }
-    }
-  }
-
-  if (state.wrapSymbol == null) return { nextIndex: -1, state: state }
-
-  switch (state.wrapSymbol) {
-    /**
-     *  - a sequence of zero or more characters between straight double-quote characters '"',
-     *    including a '"' character only if it is backslash-escaped, or
-     *  - a sequence of zero or more characters between straight single-quote characters '\'',
-     *    including a '\'' character only if it is backslash-escaped,
-     */
-    case AsciiCodePoint.DOUBLE_QUOTE:
-    case AsciiCodePoint.SINGLE_QUOTE: {
-      for (; i < endIndex; ++i) {
-        const p = nodePoints[i]
-        switch (p.codePoint) {
-          case AsciiCodePoint.BACKSLASH:
-            if (i + 1 < endIndex) {
-              state.nodePoints.push(p)
-              state.nodePoints.push(nodePoints[i + 1])
-            }
-            i += 1
-            break
-          case state.wrapSymbol:
-            // eslint-disable-next-line no-param-reassign
-            state.saturated = true
-            state.nodePoints.push(p)
-            return { nextIndex: i + 1, state: state }
-          default:
-            state.nodePoints.push(p)
-        }
-      }
-      break
-    }
-    /**
-     * a sequence of zero or more characters between matching parentheses '((...))',
-     * including a '(' or ')' character only if it is backslash-escaped.
-     */
-    case AsciiCodePoint.OPEN_PARENTHESIS: {
-      for (; i < endIndex; ++i) {
-        const p = nodePoints[i]
-        switch (p.codePoint) {
-          case AsciiCodePoint.BACKSLASH:
-            if (i + 1 < endIndex) {
-              state.nodePoints.push(p)
-              state.nodePoints.push(nodePoints[i + 1])
-            }
-            i += 1
-            break
-          case AsciiCodePoint.OPEN_PARENTHESIS:
-            return { nextIndex: -1, state: state }
-          case AsciiCodePoint.CLOSE_PARENTHESIS:
-            if (i + 1 >= endIndex || nodePoints[i + 1].codePoint === VirtualCodePoint.LINE_END) {
-              state.nodePoints.push(p)
-              // eslint-disable-next-line no-param-reassign
-              state.saturated = true
-              break
-            }
-            return { nextIndex: -1, state: state }
-          default:
-            state.nodePoints.push(p)
-        }
-      }
-      break
-    }
-  }
-
-  return { nextIndex: endIndex, state: state }
-}