@yozora/tokenizer-fenced-block 2.0.4 → 2.0.5

package/lib/cjs/{index.js → index.cjs}

@@ -86,24 +86,26 @@ function match(_api) {
 }
 
 class FencedBlockTokenizer extends coreTokenizer.BaseBlockTokenizer {
+    nodeType;
+    markers = [];
+    markersRequired;
+    checkInfoString;
     constructor(props) {
-        var _a;
         super({
             name: props.name,
-            priority: (_a = props.priority) !== null && _a !== void 0 ? _a : coreTokenizer.TokenizerPriority.FENCED_BLOCK,
+            priority: props.priority ?? coreTokenizer.TokenizerPriority.FENCED_BLOCK,
         });
-        this.markers = [];
-        this.match = match;
         this.nodeType = props.nodeType;
         this.markers = props.markers;
         this.markersRequired = props.markersRequired;
         this.checkInfoString = props.checkInfoString;
     }
+    match = match;
 }
 
 const FencedBlockType = 'fencedBlock';
 
 exports.FencedBlockTokenizer = FencedBlockTokenizer;
 exports.FencedBlockType = FencedBlockType;
-exports["default"] = FencedBlockTokenizer;
+exports.default = FencedBlockTokenizer;
 exports.fencedBlockMatch = match;

package/lib/esm/{index.js → index.mjs}

@@ -82,19 +82,21 @@ function match(_api) {
 }
 
 class FencedBlockTokenizer extends BaseBlockTokenizer {
+    nodeType;
+    markers = [];
+    markersRequired;
+    checkInfoString;
     constructor(props) {
-        var _a;
         super({
             name: props.name,
-            priority: (_a = props.priority) !== null && _a !== void 0 ? _a : TokenizerPriority.FENCED_BLOCK,
+            priority: props.priority ?? TokenizerPriority.FENCED_BLOCK,
         });
-        this.markers = [];
-        this.match = match;
         this.nodeType = props.nodeType;
         this.markers = props.markers;
         this.markersRequired = props.markersRequired;
         this.checkInfoString = props.checkInfoString;
     }
+    match = match;
 }
 
 const FencedBlockType = 'fencedBlock';

package/lib/types/index.d.ts

@@ -3,7 +3,7 @@ import { IPartialYastBlockToken, IPhrasingContentLine, ITokenizer, IBaseBlockTok
 import { INodePoint, ICodePoint } from '@yozora/character';
 
 declare const FencedBlockType = "fencedBlock";
-declare type FencedBlockType = typeof FencedBlockType;
+type FencedBlockType = typeof FencedBlockType;
 interface IToken<T extends NodeType> extends IPartialYastBlockToken<T> {
     /**
      * Line indent of a fenced block.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-fenced-block",
-  "version": "2.0.4",
+  "version": "2.0.5",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -11,33 +11,36 @@
     "directory": "tokenizers/fenced-block"
   },
   "homepage": "https://github.com/yozorajs/yozora/tree/release-2.x.x/tokenizers/fenced-block",
-  "main": "lib/cjs/index.js",
-  "module": "lib/esm/index.js",
-  "types": "lib/types/index.d.ts",
-  "source": "src/index.ts",
+  "type": "module",
+  "exports": {
+    "types": "./lib/types/index.d.ts",
+    "import": "./lib/esm/index.mjs",
+    "require": "./lib/cjs/index.cjs"
+  },
+  "source": "./src/index.ts",
+  "types": "./lib/types/index.d.ts",
+  "main": "./lib/cjs/index.cjs",
+  "module": "./lib/esm/index.mjs",
   "license": "MIT",
   "engines": {
     "node": ">= 16.0.0"
   },
   "files": [
     "lib/",
-    "!lib/**/*.js.map",
-    "!lib/**/*.d.ts.map",
+    "src/",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
     "README.md"
   ],
   "scripts": {
-    "build": "cross-env NODE_ENV=production rollup -c ../../rollup.config.js",
-    "prebuild": "rimraf lib/",
-    "prepublishOnly": "cross-env ROLLUP_SHOULD_SOURCEMAP=false yarn build",
-    "test": "echo 'No test yet.'"
+    "build": "rimraf lib/ && cross-env NODE_ENV=production rollup -c ../../rollup.config.mjs",
+    "prepublishOnly": "cross-env ROLLUP_SHOULD_SOURCEMAP=false yarn build"
   },
   "dependencies": {
-    "@yozora/ast": "^2.0.4",
-    "@yozora/character": "^2.0.4",
-    "@yozora/core-tokenizer": "^2.0.4"
+    "@yozora/ast": "^2.0.5",
+    "@yozora/character": "^2.0.5",
+    "@yozora/core-tokenizer": "^2.0.5"
   },
-  "gitHead": "c980b95254394dcacba0cbb4bea251350b09397c"
+  "gitHead": "7ba3bab49fe65cf2f57082c0503af73da9356cf0"
 }

package/src/index.ts

@@ -0,0 +1,8 @@
+export { match as fencedBlockMatch } from './match'
+export { FencedBlockTokenizer, FencedBlockTokenizer as default } from './tokenizer'
+export { FencedBlockType } from './types'
+export type {
+  IFencedBlockHookContext,
+  IToken as IFencedBlockToken,
+  ITokenizerProps as IFencedBlockTokenizerProps,
+} from './types'

package/src/match.ts

@@ -0,0 +1,170 @@
+import type { NodeType } from '@yozora/ast'
+import type { INodePoint } from '@yozora/character'
+import { calcTrimBoundaryOfCodePoints, isSpaceCharacter } from '@yozora/character'
+import type {
+  IMatchBlockHook,
+  IMatchBlockPhaseApi,
+  IPhrasingContentLine,
+  IResultOfEatAndInterruptPreviousSibling,
+  IResultOfEatContinuationText,
+  IResultOfEatOpener,
+  IYastBlockToken,
+} from '@yozora/core-tokenizer'
+import { calcEndPoint, calcStartPoint, eatOptionalCharacters } from '@yozora/core-tokenizer'
+import type { IFencedBlockHookContext, IToken } from './types'
+
+export function match<
+  T extends NodeType = NodeType,
+  IThis extends IFencedBlockHookContext<T> = IFencedBlockHookContext<T>,
+>(this: IThis, _api: IMatchBlockPhaseApi): IMatchBlockHook<T, IToken<T>> {
+  const { nodeType, markers, markersRequired, checkInfoString } = this
+  return {
+    isContainingBlock: false,
+    eatOpener,
+    eatAndInterruptPreviousSibling,
+    eatContinuationText,
+  }
+
+  function eatOpener(line: Readonly<IPhrasingContentLine>): IResultOfEatOpener<T, IToken<T>> {
+    /**
+     * Four spaces indentation produces an indented code block
+     * @see https://github.github.com/gfm/#example-104
+     */
+    if (line.countOfPrecedeSpaces >= 4) return null
+
+    const { endIndex, firstNonWhitespaceIndex } = line
+    if (firstNonWhitespaceIndex + markersRequired - 1 >= endIndex) return null
+
+    const { nodePoints, startIndex } = line
+    const marker: number = nodePoints[firstNonWhitespaceIndex].codePoint
+    if (markers.indexOf(marker) < 0) return null
+
+    const i = eatOptionalCharacters(nodePoints, firstNonWhitespaceIndex + 1, endIndex, marker)
+    const countOfMark = i - firstNonWhitespaceIndex
+
+    /**
+     * The number of marker is not enough.
+     * @see https://github.github.com/gfm/#example-91
+     */
+    if (countOfMark < markersRequired) return null
+
+    /**
+     * Eating Information string
+     * The line with the opening block fence may optionally contain some text
+     * following the block fence; this is trimmed of leading and trailing
+     * whitespace and called the info string. If the info string comes after
+     * a backtick fence, it may not contain any backtick characters. (The
+     * reason for this restriction is that otherwise some inline code would
+     * be incorrectly interpreted as the beginning of a fenced code block.)
+     * @see https://github.github.com/gfm/#info-string
+     */
+    const [iLft, iRht] = calcTrimBoundaryOfCodePoints(nodePoints, i, endIndex)
+    const infoString: INodePoint[] = nodePoints.slice(iLft, iRht)
+
+    /**
+     * Check if info string is valid, such as info strings for backtick code
+     * blocks cannot contain backticks.
+     * @see https://github.github.com/gfm/#example-115
+     * @see https://github.github.com/gfm/#example-116
+     */
+    if (checkInfoString != null && !checkInfoString(infoString, marker, countOfMark)) {
+      return null
+    }
+
+    const nextIndex = endIndex
+    const token: IToken<T> = {
+      nodeType: nodeType as T,
+      position: {
+        start: calcStartPoint(nodePoints, startIndex),
+        end: calcEndPoint(nodePoints, nextIndex - 1),
+      },
+      indent: firstNonWhitespaceIndex - startIndex,
+      marker: marker!,
+      markerCount: countOfMark,
+      lines: [],
+      infoString,
+    }
+    return { token, nextIndex }
+  }
+
+  function eatAndInterruptPreviousSibling(
+    line: Readonly<IPhrasingContentLine>,
+    prevSiblingToken: Readonly<IYastBlockToken>,
+  ): IResultOfEatAndInterruptPreviousSibling<T, IToken<T>> {
+    const result = eatOpener(line)
+    if (result == null) return null
+    return {
+      token: result.token,
+      nextIndex: result.nextIndex,
+      remainingSibling: prevSiblingToken,
+    }
+  }
+
+  function eatContinuationText(
+    line: Readonly<IPhrasingContentLine>,
+    token: IToken<T>,
+  ): IResultOfEatContinuationText {
+    const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex, countOfPrecedeSpaces } = line
+
+    /**
+     * Check closing block fence
+     *
+     * The closing block fence may be indented up to three spaces, and may be
+     * followed only by spaces, which are ignored. If the end of the containing
+     * block (or document) is reached and no closing block fence has been found,
+     * the code block contains all of the lines after the opening block fence
+     * until the end of the containing block (or document). (An alternative spec
+     * would require backtracking in the event that a closing block fence is not
+     * found. But this makes parsing much less efficient, and there seems to be
+     * no real down side to the behavior described here.)
+     * @see https://github.github.com/gfm/#code-fence
+     *
+     * Closing fence indented with at most 3 spaces
+     * @see https://github.github.com/gfm/#example-107
+     */
+    if (countOfPrecedeSpaces < 4 && firstNonWhitespaceIndex < endIndex) {
+      let i = eatOptionalCharacters(nodePoints, firstNonWhitespaceIndex, endIndex, token.marker)
+      const markerCount = i - firstNonWhitespaceIndex
+
+      /**
+       * The closing block fence must be at least as long as the opening fence
+       * @see https://github.github.com/gfm/#example-94
+       */
+      if (markerCount >= token.markerCount) {
+        // The closing block fence may be followed only by spaces.
+        for (; i < endIndex; ++i) {
+          const c = nodePoints[i].codePoint
+          if (!isSpaceCharacter(c)) break
+        }
+
+        if (i + 1 >= endIndex) {
+          return { status: 'closing', nextIndex: endIndex }
+        }
+      }
+    }
+
+    /**
+     * Eating code content
+     *
+     * The content of the code block consists of all subsequent lines, until a
+     * closing block fence of the same type as the code block began with
+     * (backticks or tildes), and with at least as many backticks or tildes as
+     * the opening block fence. If the leading block fence is indented N spaces,
+     * then up to N spaces of indentation are removed from each line of the
+     * content (if present).
+     *
+     * If a content line is not indented, it is preserved unchanged. If it is
+     * indented less than N spaces, all of the indentation is removed, but the
+     * line feed should be preserve.
+     */
+    const firstIndex = Math.min(startIndex + token.indent, firstNonWhitespaceIndex, endIndex - 1)
+    token.lines.push({
+      nodePoints,
+      startIndex: firstIndex,
+      endIndex,
+      firstNonWhitespaceIndex,
+      countOfPrecedeSpaces,
+    })
+    return { status: 'opening', nextIndex: endIndex }
+  }
+}

package/src/tokenizer.ts

@@ -0,0 +1,44 @@
+import type { Node, NodeType } from '@yozora/ast'
+import type { ICodePoint } from '@yozora/character'
+import type { IBlockTokenizer, IMatchBlockHookCreator } from '@yozora/core-tokenizer'
+import { BaseBlockTokenizer, TokenizerPriority } from '@yozora/core-tokenizer'
+import { match } from './match'
+import type { IFencedBlockHookContext, IToken, ITokenizerProps } from './types'
+
+/**
+ * Lexical Matcher for FencedBlock.
+ *
+ * A block fence is a sequence of fence-marker characters (different marker
+ * cannot be mixed.) A fenced block begins with a block fence, indented no more
+ * than three spaces.
+ *
+ * @see https://github.com/syntax-tree/mdast#code
+ * @see https://github.github.com/gfm/#code-fence
+ */
+export abstract class FencedBlockTokenizer<
+    T extends NodeType = NodeType,
+    INode extends Node<T> = Node<T>,
+    IThis extends IFencedBlockHookContext<T> = IFencedBlockHookContext<T>,
+  >
+  extends BaseBlockTokenizer<T, IToken<T>, INode, any>
+  implements IBlockTokenizer<T, IToken<T>, INode, IThis>
+{
+  protected readonly nodeType: T
+  protected readonly markers: ICodePoint[] = []
+  protected readonly markersRequired: number
+  protected readonly checkInfoString: ITokenizerProps<T>['checkInfoString']
+
+  /* istanbul ignore next */
+  constructor(props: ITokenizerProps<T>) {
+    super({
+      name: props.name,
+      priority: props.priority ?? TokenizerPriority.FENCED_BLOCK,
+    })
+    this.nodeType = props.nodeType
+    this.markers = props.markers
+    this.markersRequired = props.markersRequired
+    this.checkInfoString = props.checkInfoString
+  }
+
+  public override readonly match: IMatchBlockHookCreator<T, IToken<T>, IThis> = match
+}

package/src/types.ts

@@ -0,0 +1,91 @@
+import type { NodeType } from '@yozora/ast'
+import type { ICodePoint, INodePoint } from '@yozora/character'
+import type {
+  IBaseBlockTokenizerProps,
+  IPartialYastBlockToken,
+  IPhrasingContentLine,
+  ITokenizer,
+} from '@yozora/core-tokenizer'
+
+export const FencedBlockType = 'fencedBlock'
+// eslint-disable-next-line @typescript-eslint/no-redeclare
+export type FencedBlockType = typeof FencedBlockType
+
+export interface IToken<T extends NodeType> extends IPartialYastBlockToken<T> {
+  /**
+   * Line indent of a fenced block.
+   */
+  indent: number
+  /**
+   * Fence marker.
+   */
+  marker: number
+  /**
+   * The number of fence marker.
+   */
+  markerCount: number
+  /**
+   * Lines to construct the contents of a paragraph.
+   */
+  lines: IPhrasingContentLine[]
+  /**
+   * Meta info string
+   */
+  infoString: INodePoint[]
+}
+
+export interface IFencedBlockHookContext<T extends NodeType> extends ITokenizer {
+  /**
+   * Type of special FencedBlock token and FencedBlock node.
+   */
+  nodeType: T
+  /**
+   * Available fence markers.
+   */
+  markers: ICodePoint[]
+  /**
+   * The minimum amount required
+   */
+  markersRequired: number
+  /**
+   * Check if the info string is valid.
+   * @param infoString
+   * @param marker
+   * @param countOfMarker
+   */
+  checkInfoString?(
+    infoString: Readonly<INodePoint[]>,
+    marker: ICodePoint,
+    countOfMarker: number,
+  ): boolean
+}
+
+export interface ITokenizerProps<T extends NodeType> extends Partial<IBaseBlockTokenizerProps> {
+  /**
+   * ITokenizer name.
+   */
+  name: string
+  /**
+   * Type of special FencedBlock token and FencedBlock node.
+   */
+  nodeType: T
+  /**
+   * Available fence markers.
+   */
+  markers: ICodePoint[]
+  /**
+   * The minimum amount required
+   */
+  markersRequired: number
+  /**
+   * Check if the info string is valid.
+   * @param infoString
+   * @param marker
+   * @param countOfMarker
+   */
+  checkInfoString?(
+    infoString: Readonly<INodePoint[]>,
+    marker: ICodePoint,
+    countOfMarker: number,
+  ): boolean
+}