@yozora/tokenizer-inline-math 2.0.4 → 2.0.5-alpha.0

package/lib/cjs/{index.js → index.cjs}

@@ -150,20 +150,20 @@ const parse = function (api) {
 const uniqueName = '@yozora/tokenizer-inline-math';
 
 class InlineMathTokenizer extends coreTokenizer.BaseInlineTokenizer {
+    backtickRequired;
     constructor(props = {}) {
-        var _a, _b, _c;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.ATOMIC,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? coreTokenizer.TokenizerPriority.ATOMIC,
         });
-        this.match = match;
-        this.parse = parse;
-        this.backtickRequired = (_c = props.backtickRequired) !== null && _c !== void 0 ? _c : true;
+        this.backtickRequired = props.backtickRequired ?? true;
     }
+    match = match;
+    parse = parse;
 }
 
 exports.InlineMathTokenizer = InlineMathTokenizer;
 exports.InlineMathTokenizerName = uniqueName;
-exports["default"] = InlineMathTokenizer;
+exports.default = InlineMathTokenizer;
 exports.inlineMathMatch = match;
 exports.inlineMathParse = parse;

package/lib/esm/{index.js → index.mjs}

@@ -146,16 +146,16 @@ const parse = function (api) {
 const uniqueName = '@yozora/tokenizer-inline-math';
 
 class InlineMathTokenizer extends BaseInlineTokenizer {
+    backtickRequired;
     constructor(props = {}) {
-        var _a, _b, _c;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? TokenizerPriority.ATOMIC,
         });
-        this.match = match;
-        this.parse = parse;
-        this.backtickRequired = (_c = props.backtickRequired) !== null && _c !== void 0 ? _c : true;
+        this.backtickRequired = props.backtickRequired ?? true;
     }
+    match = match;
+    parse = parse;
 }
 
 export { InlineMathTokenizer, uniqueName as InlineMathTokenizerName, InlineMathTokenizer as default, match as inlineMathMatch, parse as inlineMathParse };

package/lib/types/index.d.ts

@@ -1,8 +1,8 @@
 import { IPartialYastInlineToken, ITokenizer, IBaseInlineTokenizerProps, IYastTokenDelimiter, IMatchInlineHookCreator, IParseInlineHookCreator, BaseInlineTokenizer, IInlineTokenizer } from '@yozora/core-tokenizer';
 import { InlineMathType, InlineMath } from '@yozora/ast';
 
-declare type T = InlineMathType;
-declare type INode = InlineMath;
+type T = InlineMathType;
+type INode = InlineMath;
 declare const uniqueName = "@yozora/tokenizer-inline-math";
 interface IToken extends IPartialYastInlineToken<T> {
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-inline-math",
-  "version": "2.0.4",
+  "version": "2.0.5-alpha.0",
   "description": "Tokenizer for processing inline math (formulas)",
   "author": {
     "name": "guanghechen",
@@ -21,36 +21,40 @@
     "inline formula",
     "inline math"
   ],
-  "main": "lib/cjs/index.js",
-  "module": "lib/esm/index.js",
-  "types": "lib/types/index.d.ts",
-  "source": "src/index.ts",
+  "type": "module",
+  "exports": {
+    "types": "./lib/types/index.d.ts",
+    "import": "./lib/esm/index.mjs",
+    "require": "./lib/cjs/index.cjs"
+  },
+  "source": "./src/index.ts",
+  "types": "./lib/types/index.d.ts",
+  "main": "./lib/cjs/index.cjs",
+  "module": "./lib/esm/index.mjs",
   "license": "MIT",
   "engines": {
     "node": ">= 16.0.0"
   },
   "files": [
     "lib/",
-    "!lib/**/*.js.map",
-    "!lib/**/*.d.ts.map",
+    "src/",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
     "README.md"
   ],
   "scripts": {
-    "build": "cross-env NODE_ENV=production rollup -c ../../rollup.config.js",
-    "prebuild": "rimraf lib/",
+    "build": "rimraf lib/ && cross-env NODE_ENV=production rollup -c ../../rollup.config.mjs",
     "prepublishOnly": "cross-env ROLLUP_SHOULD_SOURCEMAP=false yarn build",
-    "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
+    "test": "cross-env TS_NODE_FILES=true NODE_OPTIONS=--experimental-vm-modules jest --config ../../jest.config.mjs --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.0.4",
-    "@yozora/character": "^2.0.4",
-    "@yozora/core-tokenizer": "^2.0.4"
+    "@yozora/ast": "^2.0.5-alpha.0",
+    "@yozora/character": "^2.0.5-alpha.0",
+    "@yozora/core-tokenizer": "^2.0.5-alpha.0"
   },
   "devDependencies": {
-    "@yozora/tokenizer-inline-code": "^2.0.4"
+    "@yozora/tokenizer-inline-code": "^2.0.5-alpha.0"
   },
-  "gitHead": "c980b95254394dcacba0cbb4bea251350b09397c"
+  "gitHead": "8bf941fe4ef82947165b0f3cc123cd493665e13b"
 }

package/src/index.ts

@@ -0,0 +1,9 @@
+export { match as inlineMathMatch } from './match'
+export { parse as inlineMathParse } from './parse'
+export { InlineMathTokenizer, InlineMathTokenizer as default } from './tokenizer'
+export { uniqueName as InlineMathTokenizerName } from './types'
+export type {
+  IThis as IInlineMathHookContext,
+  IToken as IInlineMathToken,
+  ITokenizerProps as IInlineMathTokenizerProps,
+} from './types'

package/src/match.ts

@@ -0,0 +1,172 @@
+import { InlineMathType } from '@yozora/ast'
+import type { INodeInterval, INodePoint } from '@yozora/character'
+import { AsciiCodePoint } from '@yozora/character'
+import type {
+  IMatchInlineHookCreator,
+  IResultOfFindDelimiters,
+  IResultOfProcessSingleDelimiter,
+  IYastTokenDelimiter,
+} from '@yozora/core-tokenizer'
+import { eatOptionalCharacters } from '@yozora/core-tokenizer'
+import type { IDelimiter, IThis, IToken, T } from './types'
+
+export const match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = function (api) {
+  const { backtickRequired } = this
+  return { findDelimiter, processSingleDelimiter }
+
+  function* findDelimiter(): IResultOfFindDelimiters<IDelimiter> {
+    const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
+    const blockStartIndex: number = api.getBlockStartIndex()
+    const blockEndIndex: number = api.getBlockEndIndex()
+
+    const potentialDelimiters: IYastTokenDelimiter[] = []
+    for (let i = blockStartIndex; i < blockEndIndex; ++i) {
+      const c = nodePoints[i].codePoint
+      switch (c) {
+        case AsciiCodePoint.BACKSLASH:
+          /**
+           * Note that unlink code spans, backslash escapes works in inline-math.
+           * @see https://github.github.com/gfm/#example-348
+           */
+          i += 1
+          break
+        /**
+         * A backtick string is a string of one or more backtick characters '`'
+         * that is neither preceded nor followed by a backtick.
+         * A code span begins with a backtick string and ends with a
+         * backtick string of equal length.
+         * @see https://github.github.com/gfm/#backtick-string
+         * @see https://github.github.com/gfm/#code-span
+         *
+         * the left flanking string pattern is: <BACKTICK STRING><DOLLAR_SIGN>.
+         * eg: `$, ``$
+         *
+         * A backtick string is a string of one or more backtick
+         * characters '`' that is neither preceded nor followed by a backtick.
+         * @see https://github.github.com/gfm/#backtick-string
+         */
+        case AsciiCodePoint.BACKTICK: {
+          const _startIndex = i
+
+          // matched as many backtick as possible
+          i = eatOptionalCharacters(nodePoints, i + 1, blockEndIndex, AsciiCodePoint.BACKTICK)
+
+          // No dollar character found after backtick string
+          if (i >= blockEndIndex || nodePoints[i].codePoint !== AsciiCodePoint.DOLLAR_SIGN) {
+            break
+          }
+
+          const delimiter: IYastTokenDelimiter = {
+            type: 'opener',
+            startIndex: _startIndex,
+            endIndex: i + 1,
+          }
+          potentialDelimiters.push(delimiter)
+          break
+        }
+        /**
+         * the right flanking string pattern is: <DOLLAR_SIGN><BACKTICK STRING>.
+         * eg: $`, $``
+         *
+         * A backtick string is a string of one or more backtick characters '`'
+         * that is neither preceded nor followed by a backtick.
+         * @see https://github.github.com/gfm/#backtick-string
+         */
+        case AsciiCodePoint.DOLLAR_SIGN: {
+          // matched as many backtick as possible
+          const _startIndex = i
+          i = eatOptionalCharacters(nodePoints, i + 1, blockEndIndex, AsciiCodePoint.BACKTICK)
+
+          // A dollar sign followed by a dollar sign is not part of a valid
+          // inlineMath delimiter
+          if (i < blockEndIndex && nodePoints[i].codePoint === AsciiCodePoint.DOLLAR_SIGN) {
+            break
+          }
+
+          const thickness: number = i - _startIndex
+
+          // No backtick character found after dollar
+          if (thickness <= 1) {
+            if (backtickRequired) break
+            const delimiter: IYastTokenDelimiter = {
+              type: 'both',
+              startIndex: _startIndex,
+              endIndex: i,
+            }
+            potentialDelimiters.push(delimiter)
+            break
+          }
+
+          const delimiter: IYastTokenDelimiter = {
+            type: 'closer',
+            startIndex: _startIndex,
+            endIndex: i,
+          }
+          potentialDelimiters.push(delimiter)
+          i -= 1
+          break
+        }
+      }
+    }
+
+    let pIndex = 0
+    let lastEndIndex = -1
+    let delimiter: IDelimiter | null = null
+    while (pIndex < potentialDelimiters.length) {
+      const [startIndex, endIndex] = yield delimiter
+
+      // Read from cache.
+      if (lastEndIndex === endIndex) {
+        if (delimiter == null || delimiter.startIndex >= startIndex) continue
+      }
+      lastEndIndex = endIndex
+
+      let openerDelimiter: INodeInterval | null = null
+      let closerDelimiter: INodeInterval | null = null
+      for (; pIndex < potentialDelimiters.length; ++pIndex) {
+        for (; pIndex < potentialDelimiters.length; ++pIndex) {
+          const delimiter = potentialDelimiters[pIndex]
+          if (delimiter.startIndex >= startIndex && delimiter.type !== 'closer') break
+        }
+        if (pIndex + 1 >= potentialDelimiters.length) break
+
+        openerDelimiter = potentialDelimiters[pIndex]
+        const thickness = openerDelimiter.endIndex - openerDelimiter.startIndex
+        for (let i = pIndex + 1; i < potentialDelimiters.length; ++i) {
+          const delimiter = potentialDelimiters[i]
+          if (
+            delimiter.type !== 'opener' &&
+            delimiter.endIndex - delimiter.startIndex === thickness
+          ) {
+            closerDelimiter = delimiter
+            break
+          }
+        }
+
+        // No matched inlineCode closer marker found, try next one.
+        if (closerDelimiter != null) break
+      }
+
+      if (closerDelimiter == null) return
+
+      delimiter = {
+        type: 'full',
+        startIndex: openerDelimiter!.startIndex,
+        endIndex: closerDelimiter.endIndex,
+        thickness: closerDelimiter.endIndex - closerDelimiter.startIndex,
+      }
+    }
+  }
+
+  function processSingleDelimiter(
+    delimiter: IDelimiter,
+  ): IResultOfProcessSingleDelimiter<T, IToken> {
+    const token: IToken = {
+      nodeType: InlineMathType,
+      startIndex: delimiter.startIndex,
+      endIndex: delimiter.endIndex,
+      thickness: delimiter.thickness,
+    }
+    return [token]
+  }
+}

package/src/parse.ts

@@ -0,0 +1,53 @@
+import { InlineMathType } from '@yozora/ast'
+import type { INodePoint } from '@yozora/character'
+import { calcStringFromNodePoints, isSpaceLike } from '@yozora/character'
+import type { IParseInlineHookCreator } from '@yozora/core-tokenizer'
+import type { INode, IThis, IToken, T } from './types'
+
+export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function (api) {
+  return {
+    parse: tokens =>
+      tokens.map(token => {
+        const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
+        let startIndex: number = token.startIndex + token.thickness
+        let endIndex: number = token.endIndex - token.thickness
+
+        let isAllSpace = true
+        for (let i = startIndex; i < endIndex; ++i) {
+          if (isSpaceLike(nodePoints[i].codePoint)) continue
+          isAllSpace = false
+          break
+        }
+
+        /**
+         * If the resulting string both begins and ends with a space character,
+         * but doesn't consist entirely of space characters, a single space
+         * character is removed from the front and back. This allows you to
+         * include code that begins or endsWith backtick characters, which must
+         * be separated by whitespace from the opening or closing backtick strings.
+         * @see https://github.github.com/gfm/#example-340
+         *
+         * Only spaces, and not unicode whitespace in general, are stripped
+         * in this way
+         * @see https://github.github.com/gfm/#example-343
+         *
+         * No stripping occurs if the code span contains only spaces
+         * @see https://github.github.com/gfm/#example-344
+         */
+        if (!isAllSpace && startIndex + 2 < endIndex) {
+          const firstCharacter = nodePoints[startIndex].codePoint
+          const lastCharacter = nodePoints[endIndex - 1].codePoint
+          if (isSpaceLike(firstCharacter) && isSpaceLike(lastCharacter)) {
+            startIndex += 1
+            endIndex -= 1
+          }
+        }
+
+        const value = calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/g, ' ')
+        const node: INode = api.shouldReservePosition
+          ? { type: InlineMathType, position: api.calcPosition(token), value }
+          : { type: InlineMathType, value }
+        return node
+      }),
+  }
+}

package/src/tokenizer.ts

@@ -0,0 +1,33 @@
+import type {
+  IInlineTokenizer,
+  IMatchInlineHookCreator,
+  IParseInlineHookCreator,
+} from '@yozora/core-tokenizer'
+import { BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer'
+import { match } from './match'
+import { parse } from './parse'
+import type { IDelimiter, INode, IThis, IToken, ITokenizerProps, T } from './types'
+import { uniqueName } from './types'
+
+/**
+ * Lexical Analyzer for inlineMath.
+ */
+export class InlineMathTokenizer
+  extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
+  implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
+{
+  public readonly backtickRequired: boolean
+
+  /* istanbul ignore next */
+  constructor(props: ITokenizerProps = {}) {
+    super({
+      name: props.name ?? uniqueName,
+      priority: props.priority ?? TokenizerPriority.ATOMIC,
+    })
+    this.backtickRequired = props.backtickRequired ?? true
+  }
+
+  public override readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = match
+
+  public override readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis> = parse
+}

package/src/types.ts

@@ -0,0 +1,44 @@
+import type { InlineMath, InlineMathType } from '@yozora/ast'
+import type {
+  IBaseInlineTokenizerProps,
+  IPartialYastInlineToken,
+  ITokenizer,
+  IYastTokenDelimiter,
+} from '@yozora/core-tokenizer'
+
+export type T = InlineMathType
+export type INode = InlineMath
+export const uniqueName = '@yozora/tokenizer-inline-math'
+
+export interface IToken extends IPartialYastInlineToken<T> {
+  /**
+   * Thickness of the InlineMathDelimiter
+   */
+  thickness: number
+}
+
+/**
+ * IDelimiter of InlineMathToken.
+ */
+export interface IDelimiter extends IYastTokenDelimiter {
+  type: 'full'
+  /**
+   * Thickness of the InlineMathDelimiter
+   */
+  thickness: number
+}
+
+export interface IThis extends ITokenizer {
+  /**
+   * Whether if the backtick mark wrapping necessary.
+   */
+  readonly backtickRequired: boolean
+}
+
+export interface ITokenizerProps extends Partial<IBaseInlineTokenizerProps> {
+  /**
+   * Whether if the backtick mark wrapping necessary.
+   * @default true
+   */
+  readonly backtickRequired?: boolean
+}