@yozora/tokenizer-heading 2.0.4 → 2.0.5-alpha.0

package/lib/cjs/{index.js → index.cjs}

@@ -97,18 +97,17 @@ const uniqueName = '@yozora/tokenizer-heading';
 
 class HeadingTokenizer extends coreTokenizer.BaseBlockTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.ATOMIC,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? coreTokenizer.TokenizerPriority.ATOMIC,
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 exports.HeadingTokenizer = HeadingTokenizer;
 exports.HeadingTokenizerName = uniqueName;
-exports["default"] = HeadingTokenizer;
+exports.default = HeadingTokenizer;
 exports.headingMatch = match;
 exports.headingParse = parse;

package/lib/esm/{index.js → index.mjs}

@@ -93,14 +93,13 @@ const uniqueName = '@yozora/tokenizer-heading';
 
 class HeadingTokenizer extends BaseBlockTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? TokenizerPriority.ATOMIC,
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 export { HeadingTokenizer, uniqueName as HeadingTokenizerName, HeadingTokenizer as default, match as headingMatch, parse as headingParse };

package/lib/types/index.d.ts

@@ -1,8 +1,8 @@
 import { IPartialYastBlockToken, IPhrasingContentLine, ITokenizer, IBaseBlockTokenizerProps, IMatchBlockHookCreator, IParseBlockHookCreator, BaseBlockTokenizer, IBlockTokenizer } from '@yozora/core-tokenizer';
 import { HeadingType, Heading } from '@yozora/ast';
 
-declare type T = HeadingType;
-declare type INode = Heading;
+type T = HeadingType;
+type INode = Heading;
 declare const uniqueName = "@yozora/tokenizer-heading";
 interface IToken extends IPartialYastBlockToken<T> {
     /**
@@ -14,8 +14,8 @@ interface IToken extends IPartialYastBlockToken<T> {
      */
     line: Readonly<IPhrasingContentLine>;
 }
-declare type IThis = ITokenizer;
-declare type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
+type IThis = ITokenizer;
+type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
 
 /**
  * An ATX heading consists of a string of characters, parsed as inline content,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-heading",
-  "version": "2.0.4",
+  "version": "2.0.5-alpha.0",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -11,33 +11,37 @@
     "directory": "tokenizers/heading"
   },
   "homepage": "https://github.com/yozorajs/yozora/tree/release-2.x.x/tokenizers/heading",
-  "main": "lib/cjs/index.js",
-  "module": "lib/esm/index.js",
-  "types": "lib/types/index.d.ts",
-  "source": "src/index.ts",
+  "type": "module",
+  "exports": {
+    "types": "./lib/types/index.d.ts",
+    "import": "./lib/esm/index.mjs",
+    "require": "./lib/cjs/index.cjs"
+  },
+  "source": "./src/index.ts",
+  "types": "./lib/types/index.d.ts",
+  "main": "./lib/cjs/index.cjs",
+  "module": "./lib/esm/index.mjs",
   "license": "MIT",
   "engines": {
     "node": ">= 16.0.0"
   },
   "files": [
     "lib/",
-    "!lib/**/*.js.map",
-    "!lib/**/*.d.ts.map",
+    "src/",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
     "README.md"
   ],
   "scripts": {
-    "build": "cross-env NODE_ENV=production rollup -c ../../rollup.config.js",
-    "prebuild": "rimraf lib/",
+    "build": "rimraf lib/ && cross-env NODE_ENV=production rollup -c ../../rollup.config.mjs",
     "prepublishOnly": "cross-env ROLLUP_SHOULD_SOURCEMAP=false yarn build",
-    "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
+    "test": "cross-env TS_NODE_FILES=true NODE_OPTIONS=--experimental-vm-modules jest --config ../../jest.config.mjs --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.0.4",
-    "@yozora/character": "^2.0.4",
-    "@yozora/core-tokenizer": "^2.0.4"
+    "@yozora/ast": "^2.0.5-alpha.0",
+    "@yozora/character": "^2.0.5-alpha.0",
+    "@yozora/core-tokenizer": "^2.0.5-alpha.0"
   },
-  "gitHead": "c980b95254394dcacba0cbb4bea251350b09397c"
+  "gitHead": "8bf941fe4ef82947165b0f3cc123cd493665e13b"
 }

package/src/index.ts

@@ -0,0 +1,9 @@
+export { match as headingMatch } from './match'
+export { parse as headingParse } from './parse'
+export { HeadingTokenizer, HeadingTokenizer as default } from './tokenizer'
+export { uniqueName as HeadingTokenizerName } from './types'
+export type {
+  IThis as IHeadingHookContext,
+  IToken as IHeadingToken,
+  ITokenizerProps as IHeadingTokenizerProps,
+} from './types'

package/src/match.ts

@@ -0,0 +1,101 @@
+import { HeadingType } from '@yozora/ast'
+import { AsciiCodePoint, isSpaceCharacter } from '@yozora/character'
+import type {
+  IMatchBlockHookCreator,
+  IPhrasingContentLine,
+  IResultOfEatAndInterruptPreviousSibling,
+  IResultOfEatOpener,
+  IYastBlockToken,
+} from '@yozora/core-tokenizer'
+import { calcEndPoint, calcStartPoint, eatOptionalCharacters } from '@yozora/core-tokenizer'
+import type { IThis, IToken, T } from './types'
+
+/**
+ * An ATX heading consists of a string of characters, parsed as inline content,
+ * between an opening sequence of 1–6 unescaped '#' characters and an optional
+ * closing sequence of any number of unescaped '#' characters. The opening
+ * sequence of '#' characters must be followed by a space or by the end of line.
+ * The optional closing sequence of #s must be preceded by a space and may be
+ * followed by spaces only. The opening # character may be indented 0-3 spaces.
+ * The raw contents of the heading are stripped of leading and trailing spaces
+ * before being parsed as inline content. The heading level is equal to the
+ * number of '#' characters in the opening sequence.
+ *
+ * @see https://github.com/syntax-tree/mdast#heading
+ * @see https://github.github.com/gfm/#atx-heading
+ */
+export const match: IMatchBlockHookCreator<T, IToken, IThis> = function () {
+  return {
+    isContainingBlock: false,
+    eatOpener,
+    eatAndInterruptPreviousSibling,
+  }
+
+  function eatOpener(line: Readonly<IPhrasingContentLine>): IResultOfEatOpener<T, IToken> {
+    /**
+     * Four spaces are too much
+     * @see https://github.github.com/gfm/#example-39
+     * @see https://github.github.com/gfm/#example-40
+     */
+    if (line.countOfPrecedeSpaces >= 4) return null
+
+    const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex } = line
+    if (
+      firstNonWhitespaceIndex >= endIndex ||
+      nodePoints[firstNonWhitespaceIndex].codePoint !== AsciiCodePoint.NUMBER_SIGN
+    ) {
+      return null
+    }
+
+    const i = eatOptionalCharacters(
+      nodePoints,
+      firstNonWhitespaceIndex + 1,
+      endIndex,
+      AsciiCodePoint.NUMBER_SIGN,
+    )
+    const depth: number = i - firstNonWhitespaceIndex
+
+    /**
+     * More than six '#' characters is not a heading
+     * @see https://github.github.com/gfm/#example-33
+     */
+    if (depth > 6) return null
+
+    /**
+     * At least one space is required between the '#' characters and the
+     * heading’s contents, unless the heading is empty. Note that many
+     * implementations currently do not require the space. However, the space
+     * was required by the original ATX implementation, and it helps prevent
+     * things like the following from being parsed as headings:
+     *
+     * ATX headings can be empty
+     * @see https://github.github.com/gfm/#example-49
+     */
+    if (i + 1 < endIndex && !isSpaceCharacter(nodePoints[i].codePoint)) return null
+
+    const nextIndex = endIndex
+    const token: IToken = {
+      nodeType: HeadingType,
+      position: {
+        start: calcStartPoint(nodePoints, startIndex),
+        end: calcEndPoint(nodePoints, nextIndex - 1),
+      },
+      depth: depth as IToken['depth'],
+      line,
+    }
+    return { token, nextIndex, saturated: true }
+  }
+
+  function eatAndInterruptPreviousSibling(
+    line: Readonly<IPhrasingContentLine>,
+    prevSiblingToken: Readonly<IYastBlockToken>,
+  ): IResultOfEatAndInterruptPreviousSibling<T, IToken> {
+    const result = eatOpener(line)
+    if (result == null) return null
+    return {
+      token: result.token,
+      nextIndex: result.nextIndex,
+      remainingSibling: prevSiblingToken,
+    }
+  }
+}

package/src/parse.ts

@@ -0,0 +1,79 @@
+import type { Node } from '@yozora/ast'
+import { HeadingType } from '@yozora/ast'
+import type { INodePoint } from '@yozora/character'
+import {
+  AsciiCodePoint,
+  calcTrimBoundaryOfCodePoints,
+  isWhitespaceCharacter,
+} from '@yozora/character'
+import type { IParseBlockHookCreator, IPhrasingContentLine } from '@yozora/core-tokenizer'
+import { mergeAndStripContentLines } from '@yozora/core-tokenizer'
+import type { INode, IThis, IToken, T } from './types'
+
+export const parse: IParseBlockHookCreator<T, IToken, INode, IThis> = function (api) {
+  return {
+    parse: tokens =>
+      tokens.map(token => {
+        const { nodePoints, firstNonWhitespaceIndex, endIndex } = token.line
+
+        /**
+         * Leading and trailing whitespace is ignored in parsing inline content
+         * Spaces are allowed after the closing sequence
+         * @see https://github.github.com/gfm/#example-37
+         * @see https://github.github.com/gfm/#example-43
+         */
+        // eslint-disable-next-line prefer-const
+        let [leftIndex, rightIndex] = calcTrimBoundaryOfCodePoints(
+          nodePoints,
+          firstNonWhitespaceIndex + token.depth,
+          endIndex,
+        )
+
+        /**
+         * A closing sequence of '#' characters is optional
+         * It need not be the same length as the opening sequence
+         * @see https://github.github.com/gfm/#example-41
+         * @see https://github.github.com/gfm/#example-42
+         * @see https://github.github.com/gfm/#example-44
+         */
+        let closeCharCount = 0
+        for (let j = rightIndex - 1; j >= leftIndex; --j) {
+          const c = nodePoints[j].codePoint
+          if (c !== AsciiCodePoint.NUMBER_SIGN) break
+          closeCharCount += 1
+        }
+        if (closeCharCount > 0) {
+          let spaceCount = 0,
+            j = rightIndex - 1 - closeCharCount
+          for (; j >= leftIndex; --j) {
+            const c = nodePoints[j].codePoint
+            if (!isWhitespaceCharacter(c)) break
+            spaceCount += 1
+          }
+          if (spaceCount > 0 || j < leftIndex) {
+            rightIndex -= closeCharCount + spaceCount
+          }
+        }
+
+        // Resolve phrasing content.
+        const lines: IPhrasingContentLine[] = [
+          {
+            nodePoints,
+            startIndex: leftIndex,
+            endIndex: rightIndex,
+            firstNonWhitespaceIndex: leftIndex,
+            countOfPrecedeSpaces: 0,
+          },
+        ]
+
+        // Resolve phrasing content.
+        const contents: INodePoint[] = mergeAndStripContentLines(lines)
+        const children: Node[] = api.processInlines(contents)
+
+        const node: INode = api.shouldReservePosition
+          ? { type: HeadingType, position: token.position, depth: token.depth, children }
+          : { type: HeadingType, depth: token.depth, children }
+        return node
+      }),
+  }
+}

package/src/tokenizer.ts

@@ -0,0 +1,32 @@
+import type {
+  IBlockTokenizer,
+  IMatchBlockHookCreator,
+  IParseBlockHookCreator,
+} from '@yozora/core-tokenizer'
+import { BaseBlockTokenizer, TokenizerPriority } from '@yozora/core-tokenizer'
+import { match } from './match'
+import { parse } from './parse'
+import type { INode, IThis, IToken, ITokenizerProps, T } from './types'
+import { uniqueName } from './types'
+
+/**
+ * Lexical Analyzer for Heading.
+ * @see https://github.com/syntax-tree/mdast#heading
+ * @see https://github.github.com/gfm/#atx-heading
+ */
+export class HeadingTokenizer
+  extends BaseBlockTokenizer<T, IToken, INode, IThis>
+  implements IBlockTokenizer<T, IToken, INode, IThis>
+{
+  /* istanbul ignore next */
+  constructor(props: ITokenizerProps = {}) {
+    super({
+      name: props.name ?? uniqueName,
+      priority: props.priority ?? TokenizerPriority.ATOMIC,
+    })
+  }
+
+  public override readonly match: IMatchBlockHookCreator<T, IToken, IThis> = match
+
+  public override readonly parse: IParseBlockHookCreator<T, IToken, INode, IThis> = parse
+}

package/src/types.ts

@@ -0,0 +1,26 @@
+import type { Heading, HeadingType } from '@yozora/ast'
+import type {
+  IBaseBlockTokenizerProps,
+  IPartialYastBlockToken,
+  IPhrasingContentLine,
+  ITokenizer,
+} from '@yozora/core-tokenizer'
+
+export type T = HeadingType
+export type INode = Heading
+export const uniqueName = '@yozora/tokenizer-heading'
+
+export interface IToken extends IPartialYastBlockToken<T> {
+  /**
+   * Level of heading
+   */
+  depth: 1 | 2 | 3 | 4 | 5 | 6
+  /**
+   * Contents
+   */
+  line: Readonly<IPhrasingContentLine>
+}
+
+export type IThis = ITokenizer
+
+export type ITokenizerProps = Partial<IBaseBlockTokenizerProps>