@yozora/tokenizer-inline-math 2.1.2 → 2.1.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-inline-math",
-  "version": "2.1.2",
+  "version": "2.1.4",
   "description": "Tokenizer for processing inline math (formulas)",
   "author": {
     "name": "guanghechen",
@@ -37,7 +37,7 @@
   },
   "files": [
     "lib/",
-    "src/",
+    "lib/**/*.map",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
@@ -49,12 +49,12 @@
     "test": "cross-env TS_NODE_FILES=true NODE_OPTIONS=--experimental-vm-modules jest --config ../../jest.config.mjs --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.1.2",
-    "@yozora/character": "^2.1.2",
-    "@yozora/core-tokenizer": "^2.1.2"
+    "@yozora/ast": "^2.1.4",
+    "@yozora/character": "^2.1.4",
+    "@yozora/core-tokenizer": "^2.1.4"
   },
   "devDependencies": {
-    "@yozora/tokenizer-inline-code": "^2.1.2"
+    "@yozora/tokenizer-inline-code": "^2.1.4"
   },
-  "gitHead": "992bacafd173e7788e99fed34ce8b45f6ed24cfe"
+  "gitHead": "aa464ed1e3cd84892773a833910cfc53a556bf5f"
 }

package/src/index.ts

@@ -1,9 +0,0 @@
-export { match as inlineMathMatch } from './match'
-export { parse as inlineMathParse } from './parse'
-export { InlineMathTokenizer, InlineMathTokenizer as default } from './tokenizer'
-export { uniqueName as InlineMathTokenizerName } from './types'
-export type {
-  IThis as IInlineMathHookContext,
-  IToken as IInlineMathToken,
-  ITokenizerProps as IInlineMathTokenizerProps,
-} from './types'

package/src/match.ts

@@ -1,172 +0,0 @@
-import { InlineMathType } from '@yozora/ast'
-import type { INodeInterval, INodePoint } from '@yozora/character'
-import { AsciiCodePoint } from '@yozora/character'
-import type {
-  IMatchInlineHookCreator,
-  IResultOfFindDelimiters,
-  IResultOfProcessSingleDelimiter,
-  ITokenDelimiter,
-} from '@yozora/core-tokenizer'
-import { eatOptionalCharacters } from '@yozora/core-tokenizer'
-import type { IDelimiter, IThis, IToken, T } from './types'
-
-export const match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = function (api) {
-  const { backtickRequired } = this
-  return { findDelimiter, processSingleDelimiter }
-
-  function* findDelimiter(): IResultOfFindDelimiters<IDelimiter> {
-    const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
-    const blockStartIndex: number = api.getBlockStartIndex()
-    const blockEndIndex: number = api.getBlockEndIndex()
-
-    const potentialDelimiters: ITokenDelimiter[] = []
-    for (let i = blockStartIndex; i < blockEndIndex; ++i) {
-      const c = nodePoints[i].codePoint
-      switch (c) {
-        case AsciiCodePoint.BACKSLASH:
-          /**
-           * Note that unlink code spans, backslash escapes works in inline-math.
-           * @see https://github.github.com/gfm/#example-348
-           */
-          i += 1
-          break
-        /**
-         * A backtick string is a string of one or more backtick characters '`'
-         * that is neither preceded nor followed by a backtick.
-         * A code span begins with a backtick string and ends with a
-         * backtick string of equal length.
-         * @see https://github.github.com/gfm/#backtick-string
-         * @see https://github.github.com/gfm/#code-span
-         *
-         * the left flanking string pattern is: <BACKTICK STRING><DOLLAR_SIGN>.
-         * eg: `$, ``$
-         *
-         * A backtick string is a string of one or more backtick
-         * characters '`' that is neither preceded nor followed by a backtick.
-         * @see https://github.github.com/gfm/#backtick-string
-         */
-        case AsciiCodePoint.BACKTICK: {
-          const _startIndex = i
-
-          // matched as many backtick as possible
-          i = eatOptionalCharacters(nodePoints, i + 1, blockEndIndex, AsciiCodePoint.BACKTICK)
-
-          // No dollar character found after backtick string
-          if (i >= blockEndIndex || nodePoints[i].codePoint !== AsciiCodePoint.DOLLAR_SIGN) {
-            break
-          }
-
-          const delimiter: ITokenDelimiter = {
-            type: 'opener',
-            startIndex: _startIndex,
-            endIndex: i + 1,
-          }
-          potentialDelimiters.push(delimiter)
-          break
-        }
-        /**
-         * the right flanking string pattern is: <DOLLAR_SIGN><BACKTICK STRING>.
-         * eg: $`, $``
-         *
-         * A backtick string is a string of one or more backtick characters '`'
-         * that is neither preceded nor followed by a backtick.
-         * @see https://github.github.com/gfm/#backtick-string
-         */
-        case AsciiCodePoint.DOLLAR_SIGN: {
-          // matched as many backtick as possible
-          const _startIndex = i
-          i = eatOptionalCharacters(nodePoints, i + 1, blockEndIndex, AsciiCodePoint.BACKTICK)
-
-          // A dollar sign followed by a dollar sign is not part of a valid
-          // inlineMath delimiter
-          if (i < blockEndIndex && nodePoints[i].codePoint === AsciiCodePoint.DOLLAR_SIGN) {
-            break
-          }
-
-          const thickness: number = i - _startIndex
-
-          // No backtick character found after dollar
-          if (thickness <= 1) {
-            if (backtickRequired) break
-            const delimiter: ITokenDelimiter = {
-              type: 'both',
-              startIndex: _startIndex,
-              endIndex: i,
-            }
-            potentialDelimiters.push(delimiter)
-            break
-          }
-
-          const delimiter: ITokenDelimiter = {
-            type: 'closer',
-            startIndex: _startIndex,
-            endIndex: i,
-          }
-          potentialDelimiters.push(delimiter)
-          i -= 1
-          break
-        }
-      }
-    }
-
-    let pIndex = 0
-    let lastEndIndex = -1
-    let delimiter: IDelimiter | null = null
-    while (pIndex < potentialDelimiters.length) {
-      const [startIndex, endIndex] = yield delimiter
-
-      // Read from cache.
-      if (lastEndIndex === endIndex) {
-        if (delimiter == null || delimiter.startIndex >= startIndex) continue
-      }
-      lastEndIndex = endIndex
-
-      let openerDelimiter: INodeInterval | null = null
-      let closerDelimiter: INodeInterval | null = null
-      for (; pIndex < potentialDelimiters.length; ++pIndex) {
-        for (; pIndex < potentialDelimiters.length; ++pIndex) {
-          const delimiter = potentialDelimiters[pIndex]
-          if (delimiter.startIndex >= startIndex && delimiter.type !== 'closer') break
-        }
-        if (pIndex + 1 >= potentialDelimiters.length) break
-
-        openerDelimiter = potentialDelimiters[pIndex]
-        const thickness = openerDelimiter.endIndex - openerDelimiter.startIndex
-        for (let i = pIndex + 1; i < potentialDelimiters.length; ++i) {
-          const delimiter = potentialDelimiters[i]
-          if (
-            delimiter.type !== 'opener' &&
-            delimiter.endIndex - delimiter.startIndex === thickness
-          ) {
-            closerDelimiter = delimiter
-            break
-          }
-        }
-
-        // No matched inlineCode closer marker found, try next one.
-        if (closerDelimiter != null) break
-      }
-
-      if (closerDelimiter == null) return
-
-      delimiter = {
-        type: 'full',
-        startIndex: openerDelimiter!.startIndex,
-        endIndex: closerDelimiter.endIndex,
-        thickness: closerDelimiter.endIndex - closerDelimiter.startIndex,
-      }
-    }
-  }
-
-  function processSingleDelimiter(
-    delimiter: IDelimiter,
-  ): IResultOfProcessSingleDelimiter<T, IToken> {
-    const token: IToken = {
-      nodeType: InlineMathType,
-      startIndex: delimiter.startIndex,
-      endIndex: delimiter.endIndex,
-      thickness: delimiter.thickness,
-    }
-    return [token]
-  }
-}

package/src/parse.ts

@@ -1,53 +0,0 @@
-import { InlineMathType } from '@yozora/ast'
-import type { INodePoint } from '@yozora/character'
-import { calcStringFromNodePoints, isSpaceLike } from '@yozora/character'
-import type { IParseInlineHookCreator } from '@yozora/core-tokenizer'
-import type { INode, IThis, IToken, T } from './types'
-
-export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function (api) {
-  return {
-    parse: tokens =>
-      tokens.map(token => {
-        const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
-        let startIndex: number = token.startIndex + token.thickness
-        let endIndex: number = token.endIndex - token.thickness
-
-        let isAllSpace = true
-        for (let i = startIndex; i < endIndex; ++i) {
-          if (isSpaceLike(nodePoints[i].codePoint)) continue
-          isAllSpace = false
-          break
-        }
-
-        /**
-         * If the resulting string both begins and ends with a space character,
-         * but doesn't consist entirely of space characters, a single space
-         * character is removed from the front and back. This allows you to
-         * include code that begins or endsWith backtick characters, which must
-         * be separated by whitespace from the opening or closing backtick strings.
-         * @see https://github.github.com/gfm/#example-340
-         *
-         * Only spaces, and not unicode whitespace in general, are stripped
-         * in this way
-         * @see https://github.github.com/gfm/#example-343
-         *
-         * No stripping occurs if the code span contains only spaces
-         * @see https://github.github.com/gfm/#example-344
-         */
-        if (!isAllSpace && startIndex + 2 < endIndex) {
-          const firstCharacter = nodePoints[startIndex].codePoint
-          const lastCharacter = nodePoints[endIndex - 1].codePoint
-          if (isSpaceLike(firstCharacter) && isSpaceLike(lastCharacter)) {
-            startIndex += 1
-            endIndex -= 1
-          }
-        }
-
-        const value = calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/g, ' ')
-        const node: INode = api.shouldReservePosition
-          ? { type: InlineMathType, position: api.calcPosition(token), value }
-          : { type: InlineMathType, value }
-        return node
-      }),
-  }
-}

package/src/tokenizer.ts

@@ -1,33 +0,0 @@
-import type {
-  IInlineTokenizer,
-  IMatchInlineHookCreator,
-  IParseInlineHookCreator,
-} from '@yozora/core-tokenizer'
-import { BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer'
-import { match } from './match'
-import { parse } from './parse'
-import type { IDelimiter, INode, IThis, IToken, ITokenizerProps, T } from './types'
-import { uniqueName } from './types'
-
-/**
- * Lexical Analyzer for inlineMath.
- */
-export class InlineMathTokenizer
-  extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
-  implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
-{
-  public readonly backtickRequired: boolean
-
-  /* istanbul ignore next */
-  constructor(props: ITokenizerProps = {}) {
-    super({
-      name: props.name ?? uniqueName,
-      priority: props.priority ?? TokenizerPriority.ATOMIC,
-    })
-    this.backtickRequired = props.backtickRequired ?? true
-  }
-
-  public override readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = match
-
-  public override readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis> = parse
-}

package/src/types.ts

@@ -1,44 +0,0 @@
-import type { InlineMath, InlineMathType } from '@yozora/ast'
-import type {
-  IBaseInlineTokenizerProps,
-  IPartialInlineToken,
-  ITokenDelimiter,
-  ITokenizer,
-} from '@yozora/core-tokenizer'
-
-export type T = InlineMathType
-export type INode = InlineMath
-export const uniqueName = '@yozora/tokenizer-inline-math'
-
-export interface IToken extends IPartialInlineToken<T> {
-  /**
-   * Thickness of the InlineMathDelimiter
-   */
-  thickness: number
-}
-
-/**
- * IDelimiter of InlineMathToken.
- */
-export interface IDelimiter extends ITokenDelimiter {
-  type: 'full'
-  /**
-   * Thickness of the InlineMathDelimiter
-   */
-  thickness: number
-}
-
-export interface IThis extends ITokenizer {
-  /**
-   * Whether if the backtick mark wrapping necessary.
-   */
-  readonly backtickRequired: boolean
-}
-
-export interface ITokenizerProps extends Partial<IBaseInlineTokenizerProps> {
-  /**
-   * Whether if the backtick mark wrapping necessary.
-   * @default true
-   */
-  readonly backtickRequired?: boolean
-}