@yozora/tokenizer-thematic-break 2.0.4 → 2.0.5

package/lib/cjs/{index.js → index.cjs}

@@ -89,18 +89,17 @@ const uniqueName = '@yozora/tokenizer-thematic-break';
 
 class ThematicBreakTokenizer extends coreTokenizer.BaseBlockTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.ATOMIC,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? coreTokenizer.TokenizerPriority.ATOMIC,
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 exports.ThematicBreakTokenizer = ThematicBreakTokenizer;
 exports.ThematicBreakTokenizerName = uniqueName;
-exports["default"] = ThematicBreakTokenizer;
+exports.default = ThematicBreakTokenizer;
 exports.thematicBreakMatch = match;
 exports.thematicBreakParse = parse;

package/lib/esm/{index.js → index.mjs}

@@ -85,14 +85,13 @@ const uniqueName = '@yozora/tokenizer-thematic-break';
 
 class ThematicBreakTokenizer extends BaseBlockTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? TokenizerPriority.ATOMIC,
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 export { ThematicBreakTokenizer, uniqueName as ThematicBreakTokenizerName, ThematicBreakTokenizer as default, match as thematicBreakMatch, parse as thematicBreakParse };

package/lib/types/index.d.ts

@@ -1,8 +1,8 @@
 import { IPartialYastBlockToken, ITokenizer, IBaseBlockTokenizerProps, IMatchBlockHookCreator, IParseBlockHookCreator, BaseBlockTokenizer, IBlockTokenizer } from '@yozora/core-tokenizer';
 import { ThematicBreakType, ThematicBreak } from '@yozora/ast';
 
-declare type T = ThematicBreakType;
-declare type INode = ThematicBreak;
+type T = ThematicBreakType;
+type INode = ThematicBreak;
 declare const uniqueName = "@yozora/tokenizer-thematic-break";
 interface IToken extends IPartialYastBlockToken<T> {
     /**
@@ -14,8 +14,8 @@ interface IToken extends IPartialYastBlockToken<T> {
      */
     continuous: boolean;
 }
-declare type IThis = ITokenizer;
-declare type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
+type IThis = ITokenizer;
+type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
 
 /**
  * A line consisting of 0-3 spaces of indentation, followed by a sequence of

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-thematic-break",
-  "version": "2.0.4",
+  "version": "2.0.5",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -11,33 +11,37 @@
     "directory": "tokenizers/thematic-break"
   },
   "homepage": "https://github.com/yozorajs/yozora/tree/release-2.x.x/tokenizers/thematic-break",
-  "main": "lib/cjs/index.js",
-  "module": "lib/esm/index.js",
-  "types": "lib/types/index.d.ts",
-  "source": "src/index.ts",
+  "type": "module",
+  "exports": {
+    "types": "./lib/types/index.d.ts",
+    "import": "./lib/esm/index.mjs",
+    "require": "./lib/cjs/index.cjs"
+  },
+  "source": "./src/index.ts",
+  "types": "./lib/types/index.d.ts",
+  "main": "./lib/cjs/index.cjs",
+  "module": "./lib/esm/index.mjs",
   "license": "MIT",
   "engines": {
     "node": ">= 16.0.0"
   },
   "files": [
     "lib/",
-    "!lib/**/*.js.map",
-    "!lib/**/*.d.ts.map",
+    "src/",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
     "README.md"
   ],
   "scripts": {
-    "build": "cross-env NODE_ENV=production rollup -c ../../rollup.config.js",
-    "prebuild": "rimraf lib/",
+    "build": "rimraf lib/ && cross-env NODE_ENV=production rollup -c ../../rollup.config.mjs",
     "prepublishOnly": "cross-env ROLLUP_SHOULD_SOURCEMAP=false yarn build",
-    "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
+    "test": "cross-env TS_NODE_FILES=true NODE_OPTIONS=--experimental-vm-modules jest --config ../../jest.config.mjs --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.0.4",
-    "@yozora/character": "^2.0.4",
-    "@yozora/core-tokenizer": "^2.0.4"
+    "@yozora/ast": "^2.0.5",
+    "@yozora/character": "^2.0.5",
+    "@yozora/core-tokenizer": "^2.0.5"
   },
-  "gitHead": "c980b95254394dcacba0cbb4bea251350b09397c"
+  "gitHead": "7ba3bab49fe65cf2f57082c0503af73da9356cf0"
 }

package/src/index.ts

@@ -0,0 +1,9 @@
+export { match as thematicBreakMatch } from './match'
+export { parse as thematicBreakParse } from './parse'
+export { ThematicBreakTokenizer, ThematicBreakTokenizer as default } from './tokenizer'
+export { uniqueName as ThematicBreakTokenizerName } from './types'
+export type {
+  IThis as IThematicBreakHookContext,
+  IToken as IThematicBreakToken,
+  ITokenizerProps as IThematicBreakTokenizerProps,
+} from './types'

package/src/match.ts

@@ -0,0 +1,127 @@
+import { ThematicBreakType } from '@yozora/ast'
+import { AsciiCodePoint, isUnicodeWhitespaceCharacter } from '@yozora/character'
+import type {
+  IMatchBlockHookCreator,
+  IPhrasingContentLine,
+  IResultOfEatAndInterruptPreviousSibling,
+  IResultOfEatOpener,
+  IYastBlockToken,
+} from '@yozora/core-tokenizer'
+import { calcEndPoint, calcStartPoint } from '@yozora/core-tokenizer'
+import type { IThis, IToken, T } from './types'
+
+/**
+ * A line consisting of 0-3 spaces of indentation, followed by a sequence of
+ * three or more matching -, _, or * characters, each followed optionally by
+ * any number of spaces or tabs, forms a thematic break.
+ *
+ * @see https://github.github.com/gfm/#thematic-break
+ */
+export const match: IMatchBlockHookCreator<T, IToken, IThis> = function () {
+  return {
+    isContainingBlock: false,
+    eatOpener,
+    eatAndInterruptPreviousSibling,
+  }
+
+  function eatOpener(line: Readonly<IPhrasingContentLine>): IResultOfEatOpener<T, IToken> {
+    /**
+     * Four spaces is too much
+     * @see https://github.github.com/gfm/#example-19
+     */
+    if (line.countOfPrecedeSpaces >= 4) return null
+
+    const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex } = line
+    if (firstNonWhitespaceIndex + 2 >= endIndex) return null
+
+    let marker: number
+    let count = 0
+    let continuous = true
+    let hasPotentialInternalSpace = false
+    for (let i = firstNonWhitespaceIndex; i < endIndex; ++i) {
+      const c = nodePoints[i]
+
+      /**
+       * Spaces are allowed between the characters
+       * Spaces are allowed at the end
+       * @see https://github.github.com/gfm/#example-21
+       * @see https://github.github.com/gfm/#example-22
+       * @see https://github.github.com/gfm/#example-23
+       * @see https://github.github.com/gfm/#example-24
+       */
+      if (isUnicodeWhitespaceCharacter(c.codePoint)) {
+        hasPotentialInternalSpace = true
+        continue
+      }
+
+      /**
+       * As it is traversed from a non-empty character, if a blank character
+       * has been encountered before, it means that there is an internal space
+       */
+      if (hasPotentialInternalSpace) {
+        continuous = false
+      }
+
+      switch (c.codePoint) {
+        /**
+         * A line consisting of 0-3 spaces of indentation, followed by a
+         * sequence of three or more matching '-', '_', or '*' characters,
+         * each followed optionally by any number of spaces or tabs, forms
+         * a thematic break
+         */
+        case AsciiCodePoint.MINUS_SIGN:
+        case AsciiCodePoint.UNDERSCORE:
+        case AsciiCodePoint.ASTERISK: {
+          if (count <= 0) {
+            marker = c.codePoint
+            count += 1
+            break
+          }
+          /**
+           * It is required that all of the non-whitespace characters be the same
+           * @see https://github.github.com/gfm/#example-26
+           */
+          if (c.codePoint !== marker!) return null
+          count += 1
+          break
+        }
+        /**
+         * No other characters may occur in the line
+         * @see https://github.github.com/gfm/#example-25
+         */
+        default:
+          return null
+      }
+    }
+
+    /**
+     * Not enough characters
+     * @see https://github.github.com/gfm/#example-16
+     */
+    if (count < 3) return null
+
+    const token: IToken = {
+      nodeType: ThematicBreakType,
+      position: {
+        start: calcStartPoint(nodePoints, startIndex),
+        end: calcEndPoint(nodePoints, endIndex - 1),
+      },
+      marker: marker!,
+      continuous,
+    }
+    return { token, nextIndex: endIndex, saturated: true }
+  }
+
+  function eatAndInterruptPreviousSibling(
+    line: Readonly<IPhrasingContentLine>,
+    prevSiblingToken: Readonly<IYastBlockToken>,
+  ): IResultOfEatAndInterruptPreviousSibling<T, IToken> {
+    const result = eatOpener(line)
+    if (result == null) return null
+    return {
+      token: result.token,
+      nextIndex: result.nextIndex,
+      remainingSibling: prevSiblingToken,
+    }
+  }
+}

package/src/parse.ts

@@ -0,0 +1,15 @@
+import { ThematicBreakType } from '@yozora/ast'
+import type { IParseBlockHookCreator } from '@yozora/core-tokenizer'
+import type { INode, IThis, IToken, T } from './types'
+
+export const parse: IParseBlockHookCreator<T, IToken, INode, IThis> = function (api) {
+  return {
+    parse: tokens =>
+      tokens.map(token => {
+        const node: INode = api.shouldReservePosition
+          ? { type: ThematicBreakType, position: token.position }
+          : { type: ThematicBreakType }
+        return node
+      }),
+  }
+}

package/src/tokenizer.ts

@@ -0,0 +1,31 @@
+import type {
+  IBlockTokenizer,
+  IMatchBlockHookCreator,
+  IParseBlockHookCreator,
+} from '@yozora/core-tokenizer'
+import { BaseBlockTokenizer, TokenizerPriority } from '@yozora/core-tokenizer'
+import { match } from './match'
+import { parse } from './parse'
+import type { INode, IThis, IToken, ITokenizerProps, T } from './types'
+import { uniqueName } from './types'
+
+/**
+ * Lexical Analyzer for ThematicBreak.
+ * @see https://github.github.com/gfm/#thematic-break
+ */
+export class ThematicBreakTokenizer
+  extends BaseBlockTokenizer<T, IToken, INode, IThis>
+  implements IBlockTokenizer<T, IToken, INode, IThis>
+{
+  /* istanbul ignore next */
+  constructor(props: ITokenizerProps = {}) {
+    super({
+      name: props.name ?? uniqueName,
+      priority: props.priority ?? TokenizerPriority.ATOMIC,
+    })
+  }
+
+  public override readonly match: IMatchBlockHookCreator<T, IToken, IThis> = match
+
+  public override readonly parse: IParseBlockHookCreator<T, IToken, INode, IThis> = parse
+}

package/src/types.ts

@@ -0,0 +1,25 @@
+import type { ThematicBreak, ThematicBreakType } from '@yozora/ast'
+import type {
+  IBaseBlockTokenizerProps,
+  IPartialYastBlockToken,
+  ITokenizer,
+} from '@yozora/core-tokenizer'
+
+export type T = ThematicBreakType
+export type INode = ThematicBreak
+export const uniqueName = '@yozora/tokenizer-thematic-break'
+
+export interface IToken extends IPartialYastBlockToken<T> {
+  /**
+   * CodePoint of '-' / '_' / '*'
+   */
+  marker: number
+  /**
+   * Whether there are no internal spaces between marker characters
+   */
+  continuous: boolean
+}
+
+export type IThis = ITokenizer
+
+export type ITokenizerProps = Partial<IBaseBlockTokenizerProps>