@yozora/tokenizer-list 2.0.3 → 2.0.5-alpha.0

package/lib/cjs/{index.js → index.cjs}

@@ -255,8 +255,8 @@ const resolveList = (tokens, api) => {
         ? {
             type: ast.ListType,
             position: {
-                start: Object.assign({}, tokens[0].position.start),
-                end: Object.assign({}, tokens[tokens.length - 1].position.end),
+                start: { ...tokens[0].position.start },
+                end: { ...tokens[tokens.length - 1].position.end },
             },
             ordered: tokens[0].ordered,
             orderType: tokens[0].orderType,
@@ -281,20 +281,21 @@ const uniqueName = '@yozora/tokenizer-list';
 
 class ListTokenizer extends coreTokenizer.BaseBlockTokenizer {
     constructor(props = {}) {
-        var _a, _b, _c, _d;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.CONTAINING_BLOCK,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? coreTokenizer.TokenizerPriority.CONTAINING_BLOCK,
         });
-        this.match = match;
-        this.parse = parse;
-        this.enableTaskListItem = (_c = props.enableTaskListItem) !== null && _c !== void 0 ? _c : false;
-        this.emptyItemCouldNotInterruptedTypes = (_d = props.emptyItemCouldNotInterruptedTypes) !== null && _d !== void 0 ? _d : [
+        this.enableTaskListItem = props.enableTaskListItem ?? false;
+        this.emptyItemCouldNotInterruptedTypes = props.emptyItemCouldNotInterruptedTypes ?? [
             ast.ParagraphType,
         ];
     }
+    enableTaskListItem;
+    emptyItemCouldNotInterruptedTypes;
+    match = match;
+    parse = parse;
 }
 
 exports.ListTokenizer = ListTokenizer;
 exports.ListTokenizerName = uniqueName;
-exports["default"] = ListTokenizer;
+exports.default = ListTokenizer;

package/lib/esm/{index.js → index.mjs}

@@ -251,8 +251,8 @@ const resolveList = (tokens, api) => {
         ? {
             type: ListType,
             position: {
-                start: Object.assign({}, tokens[0].position.start),
-                end: Object.assign({}, tokens[tokens.length - 1].position.end),
+                start: { ...tokens[0].position.start },
+                end: { ...tokens[tokens.length - 1].position.end },
             },
             ordered: tokens[0].ordered,
             orderType: tokens[0].orderType,
@@ -277,18 +277,19 @@ const uniqueName = '@yozora/tokenizer-list';
 
 class ListTokenizer extends BaseBlockTokenizer {
     constructor(props = {}) {
-        var _a, _b, _c, _d;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.CONTAINING_BLOCK,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? TokenizerPriority.CONTAINING_BLOCK,
         });
-        this.match = match;
-        this.parse = parse;
-        this.enableTaskListItem = (_c = props.enableTaskListItem) !== null && _c !== void 0 ? _c : false;
-        this.emptyItemCouldNotInterruptedTypes = (_d = props.emptyItemCouldNotInterruptedTypes) !== null && _d !== void 0 ? _d : [
+        this.enableTaskListItem = props.enableTaskListItem ?? false;
+        this.emptyItemCouldNotInterruptedTypes = props.emptyItemCouldNotInterruptedTypes ?? [
             ParagraphType,
         ];
     }
+    enableTaskListItem;
+    emptyItemCouldNotInterruptedTypes;
+    match = match;
+    parse = parse;
 }
 
 export { ListTokenizer, uniqueName as ListTokenizerName, ListTokenizer as default };

package/lib/types/index.d.ts

@@ -1,8 +1,8 @@
 import { TaskStatus, NodeType, ListType, List } from '@yozora/ast';
 import { IPartialYastBlockToken, IYastBlockToken, IBaseBlockTokenizerProps, ITokenizer, BaseBlockTokenizer, IBlockTokenizer, IMatchBlockHookCreator, IParseBlockHookCreator } from '@yozora/core-tokenizer';
 
-declare type T = ListType;
-declare type INode = List;
+type T = ListType;
+type INode = List;
 declare const uniqueName = "@yozora/tokenizer-list";
 interface IToken extends IPartialYastBlockToken<T> {
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-list",
-  "version": "2.0.3",
+  "version": "2.0.5-alpha.0",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -11,33 +11,37 @@
     "directory": "tokenizers/list"
   },
   "homepage": "https://github.com/yozorajs/yozora/tree/release-2.x.x/tokenizers/list",
-  "main": "lib/cjs/index.js",
-  "module": "lib/esm/index.js",
-  "types": "lib/types/index.d.ts",
-  "source": "src/index.ts",
+  "type": "module",
+  "exports": {
+    "types": "./lib/types/index.d.ts",
+    "import": "./lib/esm/index.mjs",
+    "require": "./lib/cjs/index.cjs"
+  },
+  "source": "./src/index.ts",
+  "types": "./lib/types/index.d.ts",
+  "main": "./lib/cjs/index.cjs",
+  "module": "./lib/esm/index.mjs",
   "license": "MIT",
   "engines": {
     "node": ">= 16.0.0"
   },
   "files": [
     "lib/",
-    "!lib/**/*.js.map",
-    "!lib/**/*.d.ts.map",
+    "src/",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
     "README.md"
   ],
   "scripts": {
-    "build": "cross-env NODE_ENV=production rollup -c ../../rollup.config.js",
-    "prebuild": "rimraf lib/",
+    "build": "rimraf lib/ && cross-env NODE_ENV=production rollup -c ../../rollup.config.mjs",
     "prepublishOnly": "cross-env ROLLUP_SHOULD_SOURCEMAP=false yarn build",
-    "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
+    "test": "cross-env TS_NODE_FILES=true NODE_OPTIONS=--experimental-vm-modules jest --config ../../jest.config.mjs --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.0.3",
-    "@yozora/character": "^2.0.3",
-    "@yozora/core-tokenizer": "^2.0.3"
+    "@yozora/ast": "^2.0.5-alpha.0",
+    "@yozora/character": "^2.0.5-alpha.0",
+    "@yozora/core-tokenizer": "^2.0.5-alpha.0"
   },
-  "gitHead": "8cc8f95cfebc8d752bc3272cdd24965f540c130b"
+  "gitHead": "8bf941fe4ef82947165b0f3cc123cd493665e13b"
 }

package/src/index.ts

@@ -0,0 +1,3 @@
+export { ListTokenizer, ListTokenizer as default } from './tokenizer'
+export { uniqueName as ListTokenizerName } from './types'
+export type { IToken as IListToken, ITokenizerProps as IListTokenizerProps } from './types'

package/src/match.ts

@@ -0,0 +1,368 @@
+import { ListType, TaskStatus } from '@yozora/ast'
+import type { INodePoint } from '@yozora/character'
+import {
+  AsciiCodePoint,
+  VirtualCodePoint,
+  isAsciiDigitCharacter,
+  isAsciiLowerLetter,
+  isAsciiUpperLetter,
+  isSpaceCharacter,
+  isWhitespaceCharacter,
+} from '@yozora/character'
+import type {
+  IMatchBlockHookCreator,
+  IPhrasingContentLine,
+  IResultOfEatAndInterruptPreviousSibling,
+  IResultOfEatContinuationText,
+  IResultOfEatOpener,
+  IYastBlockToken,
+} from '@yozora/core-tokenizer'
+import { calcEndPoint, calcStartPoint } from '@yozora/core-tokenizer'
+import type { IThis, IToken, T } from './types'
+
+/**
+ * The following rules define list items:
+ *  - Basic case. If a sequence of lines Ls constitute a sequence of blocks Bs
+ *    starting with a non-whitespace character, and M is a list marker of width
+ *    W followed by 1 ≤ N ≤ 4 spaces, then the result of prepending M and the
+ *    following spaces to the first line of Ls, and indenting subsequent lines
+ *    of Ls by W + N spaces, is a list item with Bs as its contents. The type
+ *    of the list item (bullet or ordered) is determined by the type of its
+ *    list marker. If the list item is ordered, then it is also assigned a
+ *    start number, based on the ordered list marker.
+ *
+ *    Exceptions:
+ *      - When the first list item in a list interrupts a paragraph—that is,
+ *        when it starts on a line that would otherwise count as paragraph
+ *        continuation text—then
+ *        (a) the lines Ls must not begin with a blank line, and
+ *        (b) if the list item is ordered, the start number must be 1.
+ *      - If any line is a thematic break then that line is not a list item.
+ *
+ * @see https://github.com/syntax-tree/mdast#listitem
+ * @see https://github.github.com/gfm/#list-items
+ */
+
+export const match: IMatchBlockHookCreator<T, IToken, IThis> = function () {
+  const { emptyItemCouldNotInterruptedTypes, enableTaskListItem } = this
+
+  return {
+    isContainingBlock: true,
+    eatOpener,
+    eatAndInterruptPreviousSibling,
+    eatContinuationText,
+  }
+
+  function eatOpener(line: Readonly<IPhrasingContentLine>): IResultOfEatOpener<T, IToken> {
+    /**
+     * Four spaces are too much.
+     * @see https://github.github.com/gfm/#example-253
+     */
+    if (line.countOfPrecedeSpaces >= 4) return null
+
+    const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex } = line
+    if (firstNonWhitespaceIndex >= endIndex) return null
+
+    let ordered = false
+    let marker: number | null = null
+    let orderType: '1' | 'a' | 'A' | 'i' | 'I' | undefined
+    let order: number | undefined
+    let i = firstNonWhitespaceIndex
+    let c = nodePoints[i].codePoint
+
+    /**
+     * Try to resolve an ordered list-item.
+     *
+     * An ordered list marker is a sequence of 1–9 arabic digits (0-9),
+     * followed by either a . character or a ) character. (The reason
+     * for the length limit is that with 10 digits we start seeing integer
+     * overflows in some browsers.)
+     * @see https://github.github.com/gfm/#ordered-list-marker
+     *
+     * Extension: /[a-z]/ and /[A-Z]/ and [iv]+ also could be consisted the
+     * marker of an ordered list.
+     */
+    if (i + 1 < endIndex) {
+      // TODO Support roman numerals.
+      const c0 = c
+      if (isAsciiDigitCharacter(c0)) {
+        orderType = '1'
+        let v = c0 - AsciiCodePoint.DIGIT0
+        for (i += 1; i < endIndex; ++i) {
+          c = nodePoints[i].codePoint
+          if (!isAsciiDigitCharacter(c)) break
+          v = v * 10 + c - AsciiCodePoint.DIGIT0
+        }
+        order = v
+        orderType = '1'
+      } else if (isAsciiLowerLetter(c0)) {
+        i += 1
+        c = nodePoints[i].codePoint
+        order = c0 - AsciiCodePoint.LOWERCASE_A + 1
+        orderType = 'a'
+      } else if (isAsciiUpperLetter(c0)) {
+        i += 1
+        c = nodePoints[i].codePoint
+        order = c0 - AsciiCodePoint.UPPERCASE_A + 1
+        orderType = 'A'
+      }
+
+      // eat '.' / ')'
+      if (
+        i > firstNonWhitespaceIndex &&
+        i - firstNonWhitespaceIndex <= 9 &&
+        (c === AsciiCodePoint.DOT || c === AsciiCodePoint.CLOSE_PARENTHESIS)
+      ) {
+        i += 1
+        ordered = true
+        marker = c
+      }
+    }
+
+    /**
+     * Try to resolve a bullet list-item.
+     *
+     * A bullet list marker is a -, +, or * character.
+     * @see https://github.github.com/gfm/#bullet-list-marker
+     */
+    if (!ordered) {
+      if (
+        c === AsciiCodePoint.PLUS_SIGN ||
+        c === AsciiCodePoint.MINUS_SIGN ||
+        c === AsciiCodePoint.ASTERISK
+      ) {
+        i += 1
+        marker = c
+      }
+    }
+
+    if (marker == null) return null
+
+    /**
+     * When the list-item mark followed by a tab, it is treated as if it were
+     * expanded into three spaces.
+     *
+     * @see https://github.github.com/gfm/#example-7
+     */
+    let countOfSpaces = 0,
+      nextIndex = i
+    if (nextIndex < endIndex) {
+      c = nodePoints[nextIndex].codePoint
+      if (c === VirtualCodePoint.SPACE) nextIndex += 1
+    }
+
+    /**
+     * #Rule1 Basic case
+     *
+     * If a sequence of lines Ls constitute a sequence of blocks Bs starting
+     * with a non-whitespace character, and M is a list marker of width W
+     * followed by 1 ≤ N ≤ 4 spaces, then the result of prepending M and the
+     * following spaces to the first line of Ls, and indenting subsequent
+     * lines of Ls by W + N spaces, is a list item with Bs as its contents.
+     * The type of the list item (bullet or ordered) is determined by the
+     * type of its list marker. If the list item is ordered, then it is also
+     * assigned a start number, based on the ordered list marker.
+     * @see https://github.github.com/gfm/#list-items Basic case
+     */
+    for (; nextIndex < endIndex; ++nextIndex) {
+      c = nodePoints[nextIndex].codePoint
+      if (!isSpaceCharacter(c)) break
+      countOfSpaces += 1
+    }
+
+    /**
+     * Rule#2 Item starting with indented code.
+     *
+     * If a sequence of lines Ls constitute a sequence of blocks Bs starting
+     * with an indented code block, and M is a list marker of width W followed
+     * by one space, then the result of prepending M and the following space to
+     * the first line of Ls, and indenting subsequent lines of Ls by W + 1 spaces,
+     * is a list item with Bs as its contents. If a line is empty, then it need
+     * not be indented. The type of the list item (bullet or ordered) is
+     * determined by the type of its list marker. If the list item is ordered,
+     * then it is also assigned a start number, based on the ordered list marker.
+     * @see https://github.github.com/gfm/#list-items Item starting with indented code.
+     */
+    if (countOfSpaces > 4) {
+      nextIndex -= countOfSpaces - 1
+      countOfSpaces = 1
+    }
+
+    /**
+     * Rule#3 Item starting with a blank line.
+     *
+     * If a sequence of lines Ls starting with a single blank line constitute
+     * a (possibly empty) sequence of blocks Bs, not separated from each other
+     * by more than one blank line, and M is a list marker of width W, then the
+     * result of prepending M to the first line of Ls, and indenting subsequent
+     * lines of Ls by W + 1 spaces, is a list item with Bs as its contents.
+     * If a line is empty, then it need not be indented. The type of the list
+     * item (bullet or ordered) is determined by the type of its list marker.
+     * If the list item is ordered, then it is also assigned a start number,
+     * based on the ordered list marker.
+     * @see https://github.github.com/gfm/#list-items Item starting with a blank line
+     */
+    if (countOfSpaces === 0 && nextIndex < endIndex && c !== VirtualCodePoint.LINE_END) return null
+
+    const countOfTopBlankLine = c === VirtualCodePoint.LINE_END ? 1 : -1
+    if (c === VirtualCodePoint.LINE_END) {
+      nextIndex -= countOfSpaces - 1
+      countOfSpaces = 1
+    }
+
+    /**
+     * Rule#4 Indentation.
+     *
+     * If a sequence of lines Ls constitutes a list item according to rule #1,
+     * #2, or #3, then the result of indenting each line of Ls by 1-3 spaces
+     * (the same for each line) also constitutes a list item with the same
+     * contents and attributes. If a line is empty, then it need not be indented.
+     */
+    const indent = i - startIndex + countOfSpaces
+
+    // Try to resolve task status.
+    let status: TaskStatus | null = null
+    if (enableTaskListItem) {
+      ;({ status, nextIndex } = eatTaskStatus(nodePoints, nextIndex, endIndex))
+    }
+
+    const token: IToken = {
+      nodeType: ListType,
+      position: {
+        start: calcStartPoint(nodePoints, startIndex),
+        end: calcEndPoint(nodePoints, nextIndex - 1),
+      },
+      ordered,
+      marker,
+      orderType: ordered ? orderType : undefined,
+      order: ordered ? order : undefined,
+      indent,
+      countOfTopBlankLine,
+      children: [],
+    }
+
+    if (status != null) token.status = status
+    return { token, nextIndex }
+  }
+
+  function eatAndInterruptPreviousSibling(
+    line: Readonly<IPhrasingContentLine>,
+    prevSiblingToken: Readonly<IYastBlockToken>,
+  ): IResultOfEatAndInterruptPreviousSibling<T, IToken> {
+    /**
+     * ListItem can interrupt Paragraph
+     * @see https://github.github.com/gfm/#list-items Basic case Exceptions 1
+     */
+    const result = eatOpener(line)
+    if (result == null) return null
+    const { token, nextIndex } = result
+
+    /**
+     * But an empty list item cannot interrupt a paragraph
+     * @see https://github.github.com/gfm/#example-263
+     */
+    if (emptyItemCouldNotInterruptedTypes.includes(prevSiblingToken.nodeType)) {
+      if (token.indent === line.endIndex - line.startIndex) {
+        return null
+      }
+
+      /**
+       * In order to solve of unwanted lists in paragraphs with hard-wrapped
+       * numerals, we allow only lists starting with 1 to interrupt paragraphs
+       * @see https://github.github.com/gfm/#example-284
+       */
+      if (token.ordered && token.order !== 1) return null
+    }
+
+    return { token, nextIndex, remainingSibling: prevSiblingToken }
+  }
+
+  function eatContinuationText(
+    line: Readonly<IPhrasingContentLine>,
+    token: IToken,
+  ): IResultOfEatContinuationText {
+    const { startIndex, endIndex, firstNonWhitespaceIndex, countOfPrecedeSpaces: indent } = line
+
+    /**
+     * A list item can begin with at most one blank line
+     * @see https://github.github.com/gfm/#example-258
+     */
+    if (firstNonWhitespaceIndex < endIndex && indent < token.indent) {
+      return { status: 'notMatched' }
+    }
+
+    /**
+     * When encountering a blank line, it consumes at most indent characters
+     * and cannot exceed the newline character
+     * @see https://github.github.com/gfm/#example-242
+     * @see https://github.github.com/gfm/#example-298
+     */
+    if (firstNonWhitespaceIndex >= endIndex) {
+      if (token.countOfTopBlankLine >= 0) {
+        // eslint-disable-next-line no-param-reassign
+        token.countOfTopBlankLine += 1
+        if (token.countOfTopBlankLine > 1) {
+          return { status: 'notMatched' }
+        }
+      }
+    } else {
+      // eslint-disable-next-line no-param-reassign
+      token.countOfTopBlankLine = -1
+    }
+
+    const nextIndex = Math.min(startIndex + token.indent, endIndex - 1)
+    return { status: 'opening', nextIndex }
+  }
+}
+
+/**
+ * A task list item is a list item where the first block in it is a paragraph
+ * which begins with a task list item marker and at least one whitespace
+ * character before any other content.
+ *
+ * A task list item marker consists of an optional number of spaces, a left
+ * bracket ([), either a whitespace character or the letter x in either
+ * lowercase or uppercase, and then a right bracket (]).
+ *
+ * @param nodePoints
+ * @param startIndex
+ * @param endIndex
+ * @see https://github.github.com/gfm/#task-list-item
+ */
+function eatTaskStatus(
+  nodePoints: ReadonlyArray<INodePoint>,
+  startIndex: number,
+  endIndex: number,
+): { status: TaskStatus | null; nextIndex: number } {
+  let i = startIndex
+  for (; i < endIndex; ++i) {
+    const c = nodePoints[i].codePoint
+    if (!isSpaceCharacter(c)) break
+  }
+
+  if (
+    i + 3 >= endIndex ||
+    nodePoints[i].codePoint !== AsciiCodePoint.OPEN_BRACKET ||
+    nodePoints[i + 2].codePoint !== AsciiCodePoint.CLOSE_BRACKET ||
+    !isWhitespaceCharacter(nodePoints[i + 3].codePoint)
+  )
+    return { status: null, nextIndex: startIndex }
+
+  let status: TaskStatus | undefined
+  const c = nodePoints[i + 1].codePoint
+  switch (c) {
+    case AsciiCodePoint.SPACE:
+      status = TaskStatus.TODO
+      break
+    case AsciiCodePoint.MINUS_SIGN:
+      status = TaskStatus.DOING
+      break
+    case AsciiCodePoint.LOWERCASE_X:
+    case AsciiCodePoint.UPPERCASE_X:
+      status = TaskStatus.DONE
+      break
+    default:
+      return { status: null, nextIndex: startIndex }
+  }
+  return { status, nextIndex: i + 4 }
+}

package/src/parse.ts

@@ -0,0 +1,130 @@
+import type { ListItem, Node, Paragraph, Position } from '@yozora/ast'
+import { ListItemType, ListType, ParagraphType } from '@yozora/ast'
+import type { IParseBlockHookCreator, IParseBlockPhaseApi } from '@yozora/core-tokenizer'
+import type { INode, IThis, IToken, T } from './types'
+
+export const parse: IParseBlockHookCreator<T, IToken, INode, IThis> = function (api) {
+  return {
+    parse: tokens => {
+      const results: INode[] = []
+      let listItemTokens: IToken[] = []
+      for (let i = 0; i < tokens.length; ++i) {
+        const originalToken = tokens[i]
+        if (
+          listItemTokens.length <= 0 ||
+          listItemTokens[0].ordered !== originalToken.ordered ||
+          listItemTokens[0].orderType !== originalToken.orderType ||
+          listItemTokens[0].marker !== originalToken.marker
+        ) {
+          const node: INode | null = resolveList(listItemTokens, api)
+          if (node) results.push(node)
+
+          listItemTokens = [originalToken]
+          continue
+        }
+
+        /**
+         * Otherwise the current item should be a child of the originalToken,
+         * and the originalToken should be removed from the
+         * BlockTokenizerPostMatchPhaseStateTree
+         */
+        listItemTokens.push(originalToken)
+      }
+
+      const node: INode | null = resolveList(listItemTokens, api)
+      if (node) results.push(node)
+      return results
+    },
+  }
+}
+
+/**
+ * A list is loose if any of its constituent list items are separated by
+ * blank lines, or if any of its constituent list items directly contain
+ * two block-level elements with a blank line between them. Otherwise a
+ * list is tight. (The difference in HTML output is that paragraphs in a
+ * loose list are wrapped in <p> tags, while paragraphs in a tight list
+ * are not.)
+ *
+ * If list is not loose, traverse the list-items, for the list-item whose
+ * first child node is Paragraph, convert the first node in this list-item
+ * to Phrasing content
+ * @see https://github.com/syntax-tree/mdast#phrasingcontent
+ */
+const resolveList = (tokens: IToken[], api: IParseBlockPhaseApi): INode | null => {
+  if (tokens.length <= 0) return null
+
+  let spread = tokens.some((item): boolean => {
+    if (item.children == null || item.children.length <= 1) return false
+
+    let previousPosition: Position = item.children[0].position
+    for (let j = 1; j < item.children.length; ++j) {
+      const currentPosition: Position = item.children[j].position
+      if (previousPosition.end.line + 1 < currentPosition.start.line) {
+        return true
+      }
+      previousPosition = currentPosition
+    }
+    return false
+  })
+
+  if (!spread && tokens.length > 1) {
+    let previousItem = tokens[0]
+    for (let i = 1; i < tokens.length; ++i) {
+      const currentItem = tokens[i]
+
+      // If there exists blank line between list items, then the list is loose.
+      if (previousItem.position.end.line + 1 < currentItem.position.start.line) {
+        spread = true
+        break
+      }
+
+      previousItem = currentItem
+    }
+  }
+
+  const children: ListItem[] = tokens.map((listItemToken): ListItem => {
+    // Make list tighter if spread is false.
+    const nodes: Node[] = api.parseBlockTokens(listItemToken.children)
+    const children: Node[] = spread
+      ? nodes
+      : nodes
+          .map(node => (node.type === ParagraphType ? (node as Paragraph).children : node))
+          .flat()
+
+    const listItem: ListItem = api.shouldReservePosition
+      ? {
+          type: ListItemType,
+          position: listItemToken.position,
+          status: listItemToken.status,
+          children,
+        }
+      : { type: ListItemType, status: listItemToken.status, children }
+    return listItem
+  })
+
+  const node: INode = api.shouldReservePosition
+    ? {
+        type: ListType,
+        position: {
+          start: { ...tokens[0].position.start },
+          end: { ...tokens[tokens.length - 1].position.end },
+        },
+        ordered: tokens[0].ordered,
+        orderType: tokens[0].orderType,
+        start: tokens[0].order,
+        marker: tokens[0].marker,
+        spread,
+        children,
+      }
+    : {
+        type: ListType,
+        ordered: tokens[0].ordered,
+        orderType: tokens[0].orderType,
+        start: tokens[0].order,
+        marker: tokens[0].marker,
+        spread,
+        children,
+      }
+  return node
+}

package/src/tokenizer.ts

@@ -0,0 +1,45 @@
+import type { NodeType } from '@yozora/ast'
+import { ParagraphType } from '@yozora/ast'
+import type {
+  IBlockTokenizer,
+  IMatchBlockHookCreator,
+  IParseBlockHookCreator,
+} from '@yozora/core-tokenizer'
+import { BaseBlockTokenizer, TokenizerPriority } from '@yozora/core-tokenizer'
+import { match } from './match'
+import { parse } from './parse'
+import type { INode, IThis, IToken, ITokenizerProps, T } from './types'
+import { uniqueName } from './types'
+
+/**
+ * Lexical Analyzer for List.
+ *
+ * A list is a sequence of one or more list items of the same type.
+ * The list items may be separated by any number of blank lines.
+ *
+ * @see https://github.com/syntax-tree/mdast#list
+ * @see https://github.github.com/gfm/#list
+ */
+export class ListTokenizer
+  extends BaseBlockTokenizer<T, IToken, INode, IThis>
+  implements IBlockTokenizer<T, IToken, INode, IThis>
+{
+  /* istanbul ignore next */
+  constructor(props: ITokenizerProps = {}) {
+    super({
+      name: props.name ?? uniqueName,
+      priority: props.priority ?? TokenizerPriority.CONTAINING_BLOCK,
+    })
+    this.enableTaskListItem = props.enableTaskListItem ?? false
+    this.emptyItemCouldNotInterruptedTypes = props.emptyItemCouldNotInterruptedTypes ?? [
+      ParagraphType,
+    ]
+  }
+
+  public readonly enableTaskListItem: boolean
+  public readonly emptyItemCouldNotInterruptedTypes: ReadonlyArray<NodeType>
+
+  public override readonly match: IMatchBlockHookCreator<T, IToken, IThis> = match
+
+  public override readonly parse: IParseBlockHookCreator<T, IToken, INode, IThis> = parse
+}

package/src/types.ts

@@ -0,0 +1,76 @@
+import type { List, ListType, NodeType, TaskStatus } from '@yozora/ast'
+import type {
+  IBaseBlockTokenizerProps,
+  IPartialYastBlockToken,
+  ITokenizer,
+  IYastBlockToken,
+} from '@yozora/core-tokenizer'
+
+export type T = ListType
+export type INode = List
+export const uniqueName = '@yozora/tokenizer-list'
+
+export interface IToken extends IPartialYastBlockToken<T> {
+  /**
+   * Is it an ordered list item.
+   */
+  ordered: boolean
+  /**
+   * Marker of bullet list-item, or a delimiter of ordered list-item.
+   */
+  marker: number
+  /**
+   * Marker type of the list.
+   * @see https://developer.mozilla.org/en-US/docs/Web/HTML/Element/ol#attr-type
+   */
+  orderType?: '1' | 'a' | 'A' | 'i' | 'I'
+  /**
+   * Serial number of ordered list-item.
+   */
+  order?: number
+  /**
+   * Status of a todo task.
+   */
+  status?: TaskStatus
+  /**
+   * Indent of a list item.
+   */
+  indent: number
+  /**
+   * list-item 起始的空行数量
+   * The number of blank lines at the beginning of a list-item
+   */
+  countOfTopBlankLine: number
+  /**
+   * Child token nodes.
+   */
+  children: IYastBlockToken[]
+}
+
+export interface IThis extends ITokenizer {
+  /**
+   * Specify an array of Node types that could not be interrupted
+   * by this ITokenizer if the current list-item is empty.
+   * @see https://github.github.com/gfm/#example-263
+   */
+  readonly emptyItemCouldNotInterruptedTypes: ReadonlyArray<NodeType>
+
+  /**
+   * Should enable task list item (extension).
+   */
+  readonly enableTaskListItem: boolean
+}
+
+export interface ITokenizerProps extends Partial<IBaseBlockTokenizerProps> {
+  /**
+   * Specify an array of Node types that could not be interrupted
+   * by this ITokenizer if the current list-item is empty.
+   * @see https://github.github.com/gfm/#example-263
+   */
+  readonly emptyItemCouldNotInterruptedTypes?: NodeType[]
+
+  /**
+   * Should enable task list item (extension).
+   */
+  readonly enableTaskListItem?: boolean
+}