@yozora/tokenizer-link 2.1.3 → 2.1.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-link",
-  "version": "2.1.3",
+  "version": "2.1.4",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -27,7 +27,7 @@
   },
   "files": [
     "lib/",
-    "src/",
+    "lib/**/*.map",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
@@ -39,9 +39,9 @@
     "test": "cross-env TS_NODE_FILES=true NODE_OPTIONS=--experimental-vm-modules jest --config ../../jest.config.mjs --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.1.3",
-    "@yozora/character": "^2.1.3",
-    "@yozora/core-tokenizer": "^2.1.3"
+    "@yozora/ast": "^2.1.4",
+    "@yozora/character": "^2.1.4",
+    "@yozora/core-tokenizer": "^2.1.4"
   },
-  "gitHead": "9abaaff74a12d0bae0f645813241ff947a0d828c"
+  "gitHead": "aa464ed1e3cd84892773a833910cfc53a556bf5f"
 }

package/src/index.ts

@@ -1,10 +0,0 @@
-export * from './util/check-brackets'
-export * from './util/link-destination'
-export * from './util/link-title'
-export { LinkTokenizer, LinkTokenizer as default } from './tokenizer'
-export { uniqueName as LinkTokenizerName } from './types'
-export type {
-  IThis as ILinkHookContext,
-  IToken as ILinkToken,
-  ITokenizerProps as ILinkTokenizerProps,
-} from './types'

package/src/match.ts

@@ -1,207 +0,0 @@
-import { LinkType } from '@yozora/ast'
-import type { INodePoint } from '@yozora/character'
-import { AsciiCodePoint } from '@yozora/character'
-import type {
-  IInlineToken,
-  IMatchInlineHookCreator,
-  IResultOfIsDelimiterPair,
-  IResultOfProcessDelimiterPair,
-} from '@yozora/core-tokenizer'
-import { eatOptionalWhitespaces, genFindDelimiter, isLinkToken } from '@yozora/core-tokenizer'
-import type { IDelimiter, IThis, IToken, T } from './types'
-import { checkBalancedBracketsStatus } from './util/check-brackets'
-import { eatLinkDestination } from './util/link-destination'
-import { eatLinkTitle } from './util/link-title'
-
-/**
- * An inline link consists of a link text followed immediately by a left
- * parenthesis '(', optional whitespace, an optional link destination, an
- * optional link title separated from the link destination by whitespace,
- * optional whitespace, and a right parenthesis ')'. The link’s text consists
- * of the inlines contained in the link text (excluding the enclosing square
- * brackets).
- * The link’s URI consists of the link destination, excluding enclosing '<...>'
- * if present, with backslash-escapes in effect as described above. The link’s
- * title consists of the link title, excluding its enclosing delimiters, with
- * backslash-escapes in effect as described above.
- *
- * ------
- *
- * A 'opener' type delimiter is one of the following forms:
- *
- *  - '['
- *
- * A 'closer' type delimiter is one of the following forms:
- *
- *  - '](url)'
- *  - '](url "title")'
- *  - '](<url>)'
- *  - '](<url> "title")'
- *
- * @see https://github.com/syntax-tree/mdast#link
- * @see https://github.github.com/gfm/#links
- */
-export const match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = function (api) {
-  return {
-    findDelimiter: () => genFindDelimiter<IDelimiter>(_findDelimiter),
-    isDelimiterPair,
-    processDelimiterPair,
-  }
-
-  /**
-   * An inline link consists of a link text followed immediately by a left
-   * parenthesis '(', optional whitespace, an optional link destination, an
-   * optional link title separated from the link destination by whitespace,
-   * optional whitespace, and a right parenthesis ')'
-   * @see https://github.github.com/gfm/#inline-link
-   */
-  function _findDelimiter(startIndex: number, endIndex: number): IDelimiter | null {
-    const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
-    const blockEndIndex = api.getBlockEndIndex()
-
-    /**
-     * FIXME:
-     *
-     * This is a hack method to fix the situation where a higher priority token
-     * is embedded in the delimiter, at this time, ignore the tokens that have
-     * been parsed, and continue to match the content until the delimiter meets
-     * its own definition or reaches the right boundary of the block content.
-     *
-     * This algorithm has not been strictly logically verified, but I think it
-     * can work well in most cases. After all, it has passed many test cases.
-     * @see https://github.github.com/gfm/#example-588
-     */
-    for (let i = startIndex; i < endIndex; ++i) {
-      const p = nodePoints[i]
-      switch (p.codePoint) {
-        case AsciiCodePoint.BACKSLASH:
-          i += 1
-          break
-        /**
-         * A link text consists of a sequence of zero or more inline elements
-         * enclosed by square brackets ([ and ])
-         * @see https://github.github.com/gfm/#link-text
-         */
-        case AsciiCodePoint.OPEN_BRACKET: {
-          const delimiter: IDelimiter = {
-            type: 'opener',
-            startIndex: i,
-            endIndex: i + 1,
-          }
-          return delimiter
-        }
-        /**
-         * An inline link consists of a link text followed immediately by a
-         * left parenthesis '(', ..., and a right parenthesis ')'
-         * @see https://github.github.com/gfm/#inline-link
-         */
-        case AsciiCodePoint.CLOSE_BRACKET: {
-          if (i + 1 >= endIndex || nodePoints[i + 1].codePoint !== AsciiCodePoint.OPEN_PARENTHESIS)
-            break
-
-          // try to match link destination
-          const destinationStartIndex = eatOptionalWhitespaces(nodePoints, i + 2, blockEndIndex)
-          const destinationEndIndex = eatLinkDestination(
-            nodePoints,
-            destinationStartIndex,
-            blockEndIndex,
-          )
-          if (destinationEndIndex < 0) break // no valid destination matched
-
-          // try to match link title
-          const titleStartIndex = eatOptionalWhitespaces(
-            nodePoints,
-            destinationEndIndex,
-            blockEndIndex,
-          )
-          const titleEndIndex = eatLinkTitle(nodePoints, titleStartIndex, blockEndIndex)
-          if (titleEndIndex < 0) break
-
-          const _startIndex = i
-          const _endIndex = eatOptionalWhitespaces(nodePoints, titleEndIndex, blockEndIndex) + 1
-          if (
-            _endIndex > blockEndIndex ||
-            nodePoints[_endIndex - 1].codePoint !== AsciiCodePoint.CLOSE_PARENTHESIS
-          )
-            break
-
-          /**
-           * Both the title and the destination may be omitted
-           * @see https://github.github.com/gfm/#example-495
-           */
-          return {
-            type: 'closer',
-            startIndex: _startIndex,
-            endIndex: _endIndex,
-            destinationContent:
-              destinationStartIndex < destinationEndIndex
-                ? {
-                    startIndex: destinationStartIndex,
-                    endIndex: destinationEndIndex,
-                  }
-                : undefined,
-            titleContent:
-              titleStartIndex < titleEndIndex
-                ? { startIndex: titleStartIndex, endIndex: titleEndIndex }
-                : undefined,
-          }
-        }
-      }
-    }
-    return null
-  }
-
-  function isDelimiterPair(
-    openerDelimiter: IDelimiter,
-    closerDelimiter: IDelimiter,
-    internalTokens: ReadonlyArray<IInlineToken>,
-  ): IResultOfIsDelimiterPair {
-    const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
-
-    /**
-     * Links may not contain other links, at any level of nesting.
-     * @see https://github.github.com/gfm/#example-540
-     * @see https://github.github.com/gfm/#example-541
-     */
-    const hasInternalLinkToken: boolean = internalTokens.find(isLinkToken) != null
-    if (hasInternalLinkToken) {
-      return { paired: false, opener: false, closer: false }
-    }
-
-    const balancedBracketsStatus: -1 | 0 | 1 = checkBalancedBracketsStatus(
-      openerDelimiter.endIndex,
-      closerDelimiter.startIndex,
-      internalTokens,
-      nodePoints,
-    )
-    switch (balancedBracketsStatus) {
-      case -1:
-        return { paired: false, opener: false, closer: true }
-      case 0:
-        return { paired: true }
-      case 1:
-        return { paired: false, opener: true, closer: false }
-    }
-  }
-
-  function processDelimiterPair(
-    openerDelimiter: IDelimiter,
-    closerDelimiter: IDelimiter,
-    internalTokens: ReadonlyArray<IInlineToken>,
-  ): IResultOfProcessDelimiterPair<T, IToken, IDelimiter> {
-    const children: ReadonlyArray<IInlineToken> = api.resolveInternalTokens(
-      internalTokens,
-      openerDelimiter.endIndex,
-      closerDelimiter.startIndex,
-    )
-    const token: IToken = {
-      nodeType: LinkType,
-      startIndex: openerDelimiter.startIndex,
-      endIndex: closerDelimiter.endIndex,
-      destinationContent: closerDelimiter.destinationContent,
-      titleContent: closerDelimiter.titleContent,
-      children,
-    }
-    return { tokens: [token] }
-  }
-}

package/src/parse.ts

@@ -1,46 +0,0 @@
-import type { Node } from '@yozora/ast'
-import { LinkType } from '@yozora/ast'
-import type { INodePoint } from '@yozora/character'
-import { AsciiCodePoint, calcEscapedStringFromNodePoints } from '@yozora/character'
-import type { IParseInlineHookCreator } from '@yozora/core-tokenizer'
-import { encodeLinkDestination } from '@yozora/core-tokenizer'
-import type { INode, IThis, IToken, T } from './types'
-
-export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function (api) {
-  return {
-    parse: tokens =>
-      tokens.map(token => {
-        const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
-
-        // calc url
-        let url = ''
-        if (token.destinationContent != null) {
-          let { startIndex, endIndex } = token.destinationContent
-          if (nodePoints[startIndex].codePoint === AsciiCodePoint.OPEN_ANGLE) {
-            startIndex += 1
-            endIndex -= 1
-          }
-          const destination = calcEscapedStringFromNodePoints(
-            nodePoints,
-            startIndex,
-            endIndex,
-            true,
-          )
-          url = encodeLinkDestination(destination)
-        }
-
-        // calc title
-        let title: string | undefined
-        if (token.titleContent != null) {
-          const { startIndex, endIndex } = token.titleContent
-          title = calcEscapedStringFromNodePoints(nodePoints, startIndex + 1, endIndex - 1)
-        }
-
-        const children: Node[] = api.parseInlineTokens(token.children)
-        const node: INode = api.shouldReservePosition
-          ? { type: LinkType, position: api.calcPosition(token), url, title, children }
-          : { type: LinkType, url, title, children }
-        return node
-      }),
-  }
-}

package/src/tokenizer.ts

@@ -1,32 +0,0 @@
-import type {
-  IInlineTokenizer,
-  IMatchInlineHookCreator,
-  IParseInlineHookCreator,
-} from '@yozora/core-tokenizer'
-import { BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer'
-import { match } from './match'
-import { parse } from './parse'
-import type { IDelimiter, INode, IThis, IToken, ITokenizerProps, T } from './types'
-import { uniqueName } from './types'
-
-/**
- * Lexical Analyzer for InlineLink.
- * @see https://github.com/syntax-tree/mdast#link
- * @see https://github.github.com/gfm/#links
- */
-export class LinkTokenizer
-  extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
-  implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
-{
-  /* istanbul ignore next */
-  constructor(props: ITokenizerProps = {}) {
-    super({
-      name: props.name ?? uniqueName,
-      priority: props.priority ?? TokenizerPriority.LINKS,
-    })
-  }
-
-  public override readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = match
-
-  public override readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis> = parse
-}

package/src/types.ts

@@ -1,42 +0,0 @@
-import type { Link, LinkType } from '@yozora/ast'
-import type { INodeInterval } from '@yozora/character'
-import type {
-  IBaseInlineTokenizerProps,
-  IPartialInlineToken,
-  ITokenDelimiter,
-  ITokenizer,
-} from '@yozora/core-tokenizer'
-
-export type T = LinkType
-export type INode = Link
-export const uniqueName = '@yozora/tokenizer-link'
-
-export interface IToken extends IPartialInlineToken<T> {
-  /**
-   * Link destination interval.
-   */
-  destinationContent?: INodeInterval
-  /**
-   * Link title interval.
-   */
-  titleContent?: INodeInterval
-}
-
-export interface IDelimiter extends ITokenDelimiter {
-  /**
-   * IDelimiter type.
-   */
-  type: 'opener' | 'closer'
-  /**
-   * Link destination interval.
-   */
-  destinationContent?: INodeInterval
-  /**
-   * Link title interval.
-   */
-  titleContent?: INodeInterval
-}
-
-export type IThis = ITokenizer
-
-export type ITokenizerProps = Partial<IBaseInlineTokenizerProps>

package/src/util/check-brackets.ts

@@ -1,54 +0,0 @@
-import type { INodePoint } from '@yozora/character'
-import { AsciiCodePoint } from '@yozora/character'
-import type { IInlineToken } from '@yozora/core-tokenizer'
-
-/**
- * The link text may contain balanced brackets, but not unbalanced ones,
- * unless they are escaped
- *
- * @see https://github.github.com/gfm/#example-520
- * @see https://github.github.com/gfm/#example-521
- * @see https://github.github.com/gfm/#example-522
- * @see https://github.github.com/gfm/#example-523
- */
-export const checkBalancedBracketsStatus = (
-  startIndex: number,
-  endIndex: number,
-  internalTokens: ReadonlyArray<IInlineToken>,
-  nodePoints: ReadonlyArray<INodePoint>,
-): -1 | 0 | 1 => {
-  let i = startIndex
-  let bracketCount = 0
-
-  // update bracketCount
-  const updateBracketCount = (): void => {
-    const c = nodePoints[i].codePoint
-    switch (c) {
-      case AsciiCodePoint.BACKSLASH:
-        i += 1
-        break
-      case AsciiCodePoint.OPEN_BRACKET:
-        bracketCount += 1
-        break
-      case AsciiCodePoint.CLOSE_BRACKET:
-        bracketCount -= 1
-        break
-    }
-  }
-
-  for (const token of internalTokens) {
-    if (token.startIndex < startIndex) continue
-    if (token.endIndex > endIndex) break
-    for (; i < token.startIndex; ++i) {
-      updateBracketCount()
-      if (bracketCount < 0) return -1
-    }
-    i = token.endIndex
-  }
-
-  for (; i < endIndex; ++i) {
-    updateBracketCount()
-    if (bracketCount < 0) return -1
-  }
-  return bracketCount > 0 ? 1 : 0
-}

package/src/util/link-destination.ts

@@ -1,85 +0,0 @@
-import type { INodePoint } from '@yozora/character'
-import {
-  AsciiCodePoint,
-  VirtualCodePoint,
-  isAsciiControlCharacter,
-  isWhitespaceCharacter,
-} from '@yozora/character'
-
-/**
- * A link destination consists of either
- *  - a sequence of zero or more characters between an opening '<' and a closing '>'
- *    that contains no line breaks or unescaped '<' or '>' characters, or
- *  - a nonempty sequence of characters that does not start with '<', does not include
- *    ASCII space or control characters, and includes parentheses only if
- *    (a) they are backslash-escaped or
- *    (b) they are part of a balanced pair of unescaped parentheses. (Implementations
- *        may impose limits on parentheses nesting to avoid performance issues, but
- *        at least three levels of nesting should be supported.)
- * @see https://github.github.com/gfm/#link-destination
- * @return position at next iteration
- */
-export function eatLinkDestination(
-  nodePoints: ReadonlyArray<INodePoint>,
-  startIndex: number,
-  endIndex: number,
-): number {
-  if (startIndex >= endIndex) return -1
-
-  let i = startIndex
-  switch (nodePoints[i].codePoint) {
-    /**
-     * In pointy brackets:
-     *  - A sequence of zero or more characters between an opening '<' and
-     *    a closing '>' that contains no line breaks or unescaped '<' or '>' characters
-     */
-    case AsciiCodePoint.OPEN_ANGLE: {
-      for (i += 1; i < endIndex; ++i) {
-        const p = nodePoints[i]
-        switch (p.codePoint) {
-          case AsciiCodePoint.BACKSLASH:
-            i += 1
-            break
-          case AsciiCodePoint.OPEN_ANGLE:
-          case VirtualCodePoint.LINE_END:
-            return -1
-          case AsciiCodePoint.CLOSE_ANGLE:
-            return i + 1
-        }
-      }
-      return -1
-    }
-    /**
-     * Not in pointy brackets:
-     *  - A nonempty sequence of characters that does not start with '<', does not include
-     *    ASCII space or control characters, and includes parentheses only if
-     *
-     *    a) they are backslash-escaped or
-     *    b) they are part of a balanced pair of unescaped parentheses. (Implementations
-     *       may impose limits on parentheses nesting to avoid performance issues,
-     *       but at least three levels of nesting should be supported.)
-     */
-    default: {
-      let openParensCount = 0
-      for (; i < endIndex; ++i) {
-        const c = nodePoints[i].codePoint
-        switch (c) {
-          case AsciiCodePoint.BACKSLASH:
-            i += 1
-            break
-          case AsciiCodePoint.OPEN_PARENTHESIS:
-            openParensCount += 1
-            break
-          case AsciiCodePoint.CLOSE_PARENTHESIS:
-            openParensCount -= 1
-            if (openParensCount < 0) return i
-            break
-          default:
-            if (isWhitespaceCharacter(c) || isAsciiControlCharacter(c)) return i
-            break
-        }
-      }
-      return openParensCount === 0 ? i : -1
-    }
-  }
-}

package/src/util/link-title.ts

@@ -1,82 +0,0 @@
-import type { INodePoint } from '@yozora/character'
-import { AsciiCodePoint, VirtualCodePoint } from '@yozora/character'
-import { eatOptionalBlankLines } from '@yozora/core-tokenizer'
-
-/**
- * A link title consists of either
- *
- *   - a sequence of zero or more characters between straight double-quote
- *     characters '"', including a '"' character only if it is backslash-escaped, or
- *
- *   - a sequence of zero or more characters between straight single-quote
- *     characters '\'', including a '\'' character only if it is backslash-escaped, or
- *
- *   - a sequence of zero or more characters between matching parentheses '(...)',
- *     including a '(' or ')' character only if it is backslash-escaped.
- */
-export function eatLinkTitle(
-  nodePoints: ReadonlyArray<INodePoint>,
-  startIndex: number,
-  endIndex: number,
-): number {
-  if (startIndex >= endIndex) return -1
-
-  let i = startIndex
-  const titleWrapSymbol = nodePoints[i].codePoint
-  switch (titleWrapSymbol) {
-    case AsciiCodePoint.DOUBLE_QUOTE:
-    case AsciiCodePoint.SINGLE_QUOTE: {
-      for (i += 1; i < endIndex; ++i) {
-        const p = nodePoints[i]
-        switch (p.codePoint) {
-          case AsciiCodePoint.BACKSLASH:
-            i += 1
-            break
-          case titleWrapSymbol:
-            return i + 1
-          /**
-           * Although link titles may span multiple lines, they may not contain a blank line.
-           */
-          case VirtualCodePoint.LINE_END: {
-            const j = eatOptionalBlankLines(nodePoints, startIndex, i)
-            if (nodePoints[j].line > p.line + 1) return -1
-            break
-          }
-        }
-      }
-      break
-    }
-    case AsciiCodePoint.OPEN_PARENTHESIS: {
-      let openParens = 1
-      for (i += 1; i < endIndex; ++i) {
-        const p = nodePoints[i]
-        switch (p.codePoint) {
-          case AsciiCodePoint.BACKSLASH:
-            i += 1
-            break
-          /**
-           * Although link titles may span multiple lines, they may not contain a blank line.
-           */
-          case VirtualCodePoint.LINE_END: {
-            const j = eatOptionalBlankLines(nodePoints, startIndex, i)
-            if (nodePoints[j].line > p.line + 1) return -1
-            break
-          }
-          case AsciiCodePoint.OPEN_PARENTHESIS:
-            openParens += 1
-            break
-          case AsciiCodePoint.CLOSE_PARENTHESIS:
-            openParens -= 1
-            if (openParens === 0) return i + 1
-            break
-        }
-      }
-      break
-    }
-    case AsciiCodePoint.CLOSE_PARENTHESIS:
-      return i
-    default:
-      return -1
-  }
-  return -1
-}