@yozora/tokenizer-indented-code 2.0.3 → 2.0.5-alpha.0

package/lib/cjs/{index.js → index.cjs}

@@ -103,18 +103,17 @@ const uniqueName = '@yozora/tokenizer-indented-code';
 
 class IndentedCodeTokenizer extends coreTokenizer.BaseBlockTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.ATOMIC,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? coreTokenizer.TokenizerPriority.ATOMIC,
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 exports.IndentedCodeTokenizer = IndentedCodeTokenizer;
 exports.IndentedCodeTokenizerName = uniqueName;
-exports["default"] = IndentedCodeTokenizer;
+exports.default = IndentedCodeTokenizer;
 exports.indentedCodeMatch = match;
 exports.indentedCodeParse = parse;

package/lib/esm/{index.js → index.mjs}

@@ -99,14 +99,13 @@ const uniqueName = '@yozora/tokenizer-indented-code';
 
 class IndentedCodeTokenizer extends BaseBlockTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? TokenizerPriority.ATOMIC,
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 export { IndentedCodeTokenizer, uniqueName as IndentedCodeTokenizerName, IndentedCodeTokenizer as default, match as indentedCodeMatch, parse as indentedCodeParse };

package/lib/types/index.d.ts

@@ -1,8 +1,8 @@
 import { IPartialYastBlockToken, IPhrasingContentLine, ITokenizer, IBaseBlockTokenizerProps, IMatchBlockHookCreator, IParseBlockHookCreator, BaseBlockTokenizer, IBlockTokenizer } from '@yozora/core-tokenizer';
 import { CodeType, Code } from '@yozora/ast';
 
-declare type T = CodeType;
-declare type INode = Code;
+type T = CodeType;
+type INode = Code;
 declare const uniqueName = "@yozora/tokenizer-indented-code";
 interface IToken extends IPartialYastBlockToken<T> {
     /**
@@ -10,8 +10,8 @@ interface IToken extends IPartialYastBlockToken<T> {
      */
     lines: IPhrasingContentLine[];
 }
-declare type IThis = ITokenizer;
-declare type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
+type IThis = ITokenizer;
+type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
 
 /**
  * An indented code block is composed of one or more indented chunks

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-indented-code",
-  "version": "2.0.3",
+  "version": "2.0.5-alpha.0",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -11,33 +11,37 @@
     "directory": "tokenizers/indented-code"
   },
   "homepage": "https://github.com/yozorajs/yozora/tree/release-2.x.x/tokenizers/indented-code",
-  "main": "lib/cjs/index.js",
-  "module": "lib/esm/index.js",
-  "types": "lib/types/index.d.ts",
-  "source": "src/index.ts",
+  "type": "module",
+  "exports": {
+    "types": "./lib/types/index.d.ts",
+    "import": "./lib/esm/index.mjs",
+    "require": "./lib/cjs/index.cjs"
+  },
+  "source": "./src/index.ts",
+  "types": "./lib/types/index.d.ts",
+  "main": "./lib/cjs/index.cjs",
+  "module": "./lib/esm/index.mjs",
   "license": "MIT",
   "engines": {
     "node": ">= 16.0.0"
   },
   "files": [
     "lib/",
-    "!lib/**/*.js.map",
-    "!lib/**/*.d.ts.map",
+    "src/",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
     "README.md"
   ],
   "scripts": {
-    "build": "cross-env NODE_ENV=production rollup -c ../../rollup.config.js",
-    "prebuild": "rimraf lib/",
+    "build": "rimraf lib/ && cross-env NODE_ENV=production rollup -c ../../rollup.config.mjs",
     "prepublishOnly": "cross-env ROLLUP_SHOULD_SOURCEMAP=false yarn build",
-    "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
+    "test": "cross-env TS_NODE_FILES=true NODE_OPTIONS=--experimental-vm-modules jest --config ../../jest.config.mjs --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.0.3",
-    "@yozora/character": "^2.0.3",
-    "@yozora/core-tokenizer": "^2.0.3"
+    "@yozora/ast": "^2.0.5-alpha.0",
+    "@yozora/character": "^2.0.5-alpha.0",
+    "@yozora/core-tokenizer": "^2.0.5-alpha.0"
   },
-  "gitHead": "8cc8f95cfebc8d752bc3272cdd24965f540c130b"
+  "gitHead": "8bf941fe4ef82947165b0f3cc123cd493665e13b"
 }

package/src/index.ts

@@ -0,0 +1,9 @@
+export { match as indentedCodeMatch } from './match'
+export { parse as indentedCodeParse } from './parse'
+export { IndentedCodeTokenizer, IndentedCodeTokenizer as default } from './tokenizer'
+export { uniqueName as IndentedCodeTokenizerName } from './types'
+export type {
+  IThis as IIndentedCodeHookContext,
+  IToken as IIndentedCodeToken,
+  ITokenizerProps as IIndentedCodeTokenizerProps,
+} from './types'

package/src/match.ts

@@ -0,0 +1,94 @@
+import { CodeType } from '@yozora/ast'
+import { AsciiCodePoint, VirtualCodePoint } from '@yozora/character'
+import type {
+  IMatchBlockHookCreator,
+  IPhrasingContentLine,
+  IResultOfEatContinuationText,
+  IResultOfEatOpener,
+} from '@yozora/core-tokenizer'
+import { calcEndPoint, calcStartPoint } from '@yozora/core-tokenizer'
+import type { IThis, IToken, T } from './types'
+
+/**
+ * An indented code block is composed of one or more indented chunks
+ * separated by blank lines. An indented chunk is a sequence of non-blank
+ * lines, each indented four or more spaces. The contents of the code block
+ * are the literal contents of the lines, including trailing line endings,
+ * minus four spaces of indentation.
+ *
+ * @see https://github.github.com/gfm/#indented-code-block
+ */
+export const match: IMatchBlockHookCreator<T, IToken, IThis> = function () {
+  return {
+    isContainingBlock: false,
+    eatOpener,
+    eatContinuationText,
+  }
+
+  function eatOpener(line: Readonly<IPhrasingContentLine>): IResultOfEatOpener<T, IToken> {
+    if (line.countOfPrecedeSpaces < 4) return null
+    const { nodePoints, startIndex, firstNonWhitespaceIndex, endIndex } = line
+
+    let firstIndex = startIndex + 4
+
+    /**
+     * If there exists 1-3 spaces before a tab forms the indent, the remain
+     * virtual spaces of the tab should not be a part of the contents.
+     * @see https://github.github.com/gfm/#example-2
+     */
+    if (
+      nodePoints[startIndex].codePoint === AsciiCodePoint.SPACE &&
+      nodePoints[startIndex + 3].codePoint === VirtualCodePoint.SPACE
+    ) {
+      let i = startIndex + 1
+      for (; i < firstNonWhitespaceIndex; ++i) {
+        if (nodePoints[i].codePoint === VirtualCodePoint.SPACE) break
+      }
+      firstIndex = i + 4
+    }
+
+    const nextIndex = endIndex
+    const token: IToken = {
+      nodeType: CodeType,
+      position: {
+        start: calcStartPoint(nodePoints, startIndex),
+        end: calcEndPoint(nodePoints, nextIndex - 1),
+      },
+      lines: [
+        {
+          nodePoints,
+          startIndex: firstIndex,
+          endIndex,
+          firstNonWhitespaceIndex,
+          countOfPrecedeSpaces: line.countOfPrecedeSpaces - (firstIndex - startIndex),
+        },
+      ],
+    }
+    return { token, nextIndex }
+  }
+
+  function eatContinuationText(
+    line: Readonly<IPhrasingContentLine>,
+    token: IToken,
+  ): IResultOfEatContinuationText {
+    const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex, countOfPrecedeSpaces } = line
+
+    if (countOfPrecedeSpaces < 4 && firstNonWhitespaceIndex < endIndex)
+      return { status: 'notMatched' }
+
+    /**
+     * Blank line is allowed
+     * @see https://github.github.com/gfm/#example-81
+     * @see https://github.github.com/gfm/#example-82
+     */
+    const firstIndex = Math.min(endIndex - 1, startIndex + 4)
+    token.lines.push({
+      nodePoints,
+      startIndex: firstIndex,
+      endIndex,
+      firstNonWhitespaceIndex,
+      countOfPrecedeSpaces: countOfPrecedeSpaces - (firstIndex - startIndex),
+    })
+    return { status: 'opening', nextIndex: endIndex }
+  }
+}

package/src/parse.ts

@@ -0,0 +1,54 @@
+import { CodeType } from '@yozora/ast'
+import type { INodePoint } from '@yozora/character'
+import { calcStringFromNodePoints } from '@yozora/character'
+import type { IParseBlockHookCreator } from '@yozora/core-tokenizer'
+import { mergeContentLinesFaithfully } from '@yozora/core-tokenizer'
+import type { INode, IThis, IToken, T } from './types'
+
+export const parse: IParseBlockHookCreator<T, IToken, INode, IThis> = function (api) {
+  return {
+    parse: tokens =>
+      tokens.map(token => {
+        /**
+         * Blank lines preceding or following an indented code block
+         * are not included in it
+         * @see https://github.github.com/gfm/#example-87
+         */
+        const { lines } = token
+        let startLineIndex = 0,
+          endLineIndex = lines.length
+        for (; startLineIndex < endLineIndex; ++startLineIndex) {
+          const line = lines[startLineIndex]
+          if (line.firstNonWhitespaceIndex < line.endIndex) break
+        }
+        for (; startLineIndex < endLineIndex; --endLineIndex) {
+          const line = lines[endLineIndex - 1]
+          if (line.firstNonWhitespaceIndex < line.endIndex) break
+        }
+
+        const contents: INodePoint[] = mergeContentLinesFaithfully(
+          lines,
+          startLineIndex,
+          endLineIndex,
+        )
+
+        let value: string = calcStringFromNodePoints(contents)
+        if (!/\n$/.test(value)) value += '\n'
+        const node: INode = api.shouldReservePosition
+          ? {
+              type: CodeType,
+              position: token.position,
+              lang: null,
+              meta: null,
+              value,
+            }
+          : {
+              type: CodeType,
+              lang: null,
+              meta: null,
+              value,
+            }
+        return node
+      }),
+  }
+}

package/src/tokenizer.ts

@@ -0,0 +1,31 @@
+import type {
+  IBlockTokenizer,
+  IMatchBlockHookCreator,
+  IParseBlockHookCreator,
+} from '@yozora/core-tokenizer'
+import { BaseBlockTokenizer, TokenizerPriority } from '@yozora/core-tokenizer'
+import { match } from './match'
+import { parse } from './parse'
+import type { INode, IThis, IToken, ITokenizerProps, T } from './types'
+import { uniqueName } from './types'
+
+/**
+ * Lexical Analyzer for IndentedCode.
+ * @see https://github.github.com/gfm/#indented-code-block
+ */
+export class IndentedCodeTokenizer
+  extends BaseBlockTokenizer<T, IToken, INode, IThis>
+  implements IBlockTokenizer<T, IToken, INode, IThis>
+{
+  /* istanbul ignore next */
+  constructor(props: ITokenizerProps = {}) {
+    super({
+      name: props.name ?? uniqueName,
+      priority: props.priority ?? TokenizerPriority.ATOMIC,
+    })
+  }
+
+  public override readonly match: IMatchBlockHookCreator<T, IToken, IThis> = match
+
+  public override readonly parse: IParseBlockHookCreator<T, IToken, INode, IThis> = parse
+}

package/src/types.ts

@@ -0,0 +1,22 @@
+import type { Code, CodeType } from '@yozora/ast'
+import type {
+  IBaseBlockTokenizerProps,
+  IPartialYastBlockToken,
+  IPhrasingContentLine,
+  ITokenizer,
+} from '@yozora/core-tokenizer'
+
+export type T = CodeType
+export type INode = Code
+export const uniqueName = '@yozora/tokenizer-indented-code'
+
+export interface IToken extends IPartialYastBlockToken<T> {
+  /**
+   * Lines to construct the contents of a paragraph.
+   */
+  lines: IPhrasingContentLine[]
+}
+
+export type IThis = ITokenizer
+
+export type ITokenizerProps = Partial<IBaseBlockTokenizerProps>