@yozora/tokenizer-fenced-code 2.0.3 → 2.0.5-alpha.0

package/lib/cjs/{index.js → index.cjs}

@@ -7,13 +7,12 @@ var ast = require('@yozora/ast');
 var character = require('@yozora/character');
 var coreTokenizer = require('@yozora/core-tokenizer');
 
-function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
-
-var FencedBlockTokenizer__default = /*#__PURE__*/_interopDefaultLegacy(FencedBlockTokenizer);
-
 const match = function (api) {
     const hook = FencedBlockTokenizer.fencedBlockMatch.call(this, api);
-    return Object.assign(Object.assign({}, hook), { isContainingBlock: false });
+    return {
+        ...hook,
+        isContainingBlock: false,
+    };
 };
 
 const parse = function (api) {
@@ -56,12 +55,11 @@ const parse = function (api) {
 
 const uniqueName = '@yozora/tokenizer-fenced-code';
 
-class FencedCodeTokenizer extends FencedBlockTokenizer__default["default"] {
+class FencedCodeTokenizer extends FencedBlockTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.FENCED_BLOCK,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? coreTokenizer.TokenizerPriority.FENCED_BLOCK,
             nodeType: ast.CodeType,
             markers: [character.AsciiCodePoint.BACKTICK, character.AsciiCodePoint.TILDE],
             markersRequired: 3,
@@ -75,13 +73,13 @@ class FencedCodeTokenizer extends FencedBlockTokenizer__default["default"] {
                 return true;
             },
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 exports.FencedCodeTokenizer = FencedCodeTokenizer;
 exports.FencedCodeTokenizerName = uniqueName;
-exports["default"] = FencedCodeTokenizer;
+exports.default = FencedCodeTokenizer;
 exports.fencedCodeMatch = match;
 exports.fencedCodeParse = parse;

package/lib/esm/{index.js → index.mjs}

@@ -5,7 +5,10 @@ import { eatOptionalWhitespaces, mergeContentLinesFaithfully, TokenizerPriority
 
 const match = function (api) {
     const hook = fencedBlockMatch.call(this, api);
-    return Object.assign(Object.assign({}, hook), { isContainingBlock: false });
+    return {
+        ...hook,
+        isContainingBlock: false,
+    };
 };
 
 const parse = function (api) {
@@ -50,10 +53,9 @@ const uniqueName = '@yozora/tokenizer-fenced-code';
 
 class FencedCodeTokenizer extends FencedBlockTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.FENCED_BLOCK,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? TokenizerPriority.FENCED_BLOCK,
             nodeType: CodeType,
             markers: [AsciiCodePoint.BACKTICK, AsciiCodePoint.TILDE],
             markersRequired: 3,
@@ -67,9 +69,9 @@ class FencedCodeTokenizer extends FencedBlockTokenizer {
                 return true;
             },
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 export { FencedCodeTokenizer, uniqueName as FencedCodeTokenizerName, FencedCodeTokenizer as default, match as fencedCodeMatch, parse as fencedCodeParse };

package/lib/types/index.d.ts

@@ -2,12 +2,12 @@ import { IBaseBlockTokenizerProps, IMatchBlockHookCreator, IParseBlockHookCreato
 import { CodeType, Code } from '@yozora/ast';
 import FencedBlockTokenizer, { IFencedBlockToken, IFencedBlockHookContext } from '@yozora/tokenizer-fenced-block';
 
-declare type T = CodeType;
-declare type INode = Code;
+type T = CodeType;
+type INode = Code;
 declare const uniqueName = "@yozora/tokenizer-fenced-code";
-declare type IToken = IFencedBlockToken<T>;
-declare type IThis = IFencedBlockHookContext<T>;
-declare type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
+type IToken = IFencedBlockToken<T>;
+type IThis = IFencedBlockHookContext<T>;
+type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
 
 /**
  * A code fence is a sequence of at least three consecutive backtick characters

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-fenced-code",
-  "version": "2.0.3",
+  "version": "2.0.5-alpha.0",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -11,34 +11,38 @@
     "directory": "tokenizers/fenced-code"
   },
   "homepage": "https://github.com/yozorajs/yozora/tree/release-2.x.x/tokenizers/fenced-code",
-  "main": "lib/cjs/index.js",
-  "module": "lib/esm/index.js",
-  "types": "lib/types/index.d.ts",
-  "source": "src/index.ts",
+  "type": "module",
+  "exports": {
+    "types": "./lib/types/index.d.ts",
+    "import": "./lib/esm/index.mjs",
+    "require": "./lib/cjs/index.cjs"
+  },
+  "source": "./src/index.ts",
+  "types": "./lib/types/index.d.ts",
+  "main": "./lib/cjs/index.cjs",
+  "module": "./lib/esm/index.mjs",
   "license": "MIT",
   "engines": {
     "node": ">= 16.0.0"
   },
   "files": [
     "lib/",
-    "!lib/**/*.js.map",
-    "!lib/**/*.d.ts.map",
+    "src/",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
     "README.md"
   ],
   "scripts": {
-    "build": "cross-env NODE_ENV=production rollup -c ../../rollup.config.js",
-    "prebuild": "rimraf lib/",
+    "build": "rimraf lib/ && cross-env NODE_ENV=production rollup -c ../../rollup.config.mjs",
     "prepublishOnly": "cross-env ROLLUP_SHOULD_SOURCEMAP=false yarn build",
-    "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
+    "test": "cross-env TS_NODE_FILES=true NODE_OPTIONS=--experimental-vm-modules jest --config ../../jest.config.mjs --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.0.3",
-    "@yozora/character": "^2.0.3",
-    "@yozora/core-tokenizer": "^2.0.3",
-    "@yozora/tokenizer-fenced-block": "^2.0.3"
+    "@yozora/ast": "^2.0.5-alpha.0",
+    "@yozora/character": "^2.0.5-alpha.0",
+    "@yozora/core-tokenizer": "^2.0.5-alpha.0",
+    "@yozora/tokenizer-fenced-block": "^2.0.5-alpha.0"
   },
-  "gitHead": "8cc8f95cfebc8d752bc3272cdd24965f540c130b"
+  "gitHead": "8bf941fe4ef82947165b0f3cc123cd493665e13b"
 }

package/src/index.ts

@@ -0,0 +1,9 @@
+export { match as fencedCodeMatch } from './match'
+export { parse as fencedCodeParse } from './parse'
+export { FencedCodeTokenizer, FencedCodeTokenizer as default } from './tokenizer'
+export { uniqueName as FencedCodeTokenizerName } from './types'
+export type {
+  IThis as IFencedCodeHookContext,
+  IToken as IFencedCodeToken,
+  ITokenizerProps as IFencedCodeTokenizerProps,
+} from './types'

package/src/match.ts

@@ -0,0 +1,19 @@
+import type { IMatchBlockHook, IMatchBlockHookCreator } from '@yozora/core-tokenizer'
+import { fencedBlockMatch } from '@yozora/tokenizer-fenced-block'
+import type { IThis, IToken, T } from './types'
+
+/**
+ * A code fence is a sequence of at least three consecutive backtick characters
+ * (`) or tildes (~). (Tildes and backticks cannot be mixed.) A fenced code
+ * block begins with a code fence, indented no more than three spaces.
+ *
+ * @see https://github.com/syntax-tree/mdast#code
+ * @see https://github.github.com/gfm/#code-fence
+ */
+export const match: IMatchBlockHookCreator<T, IToken, IThis> = function (api) {
+  const hook = fencedBlockMatch.call(this, api) as IMatchBlockHook<T, IToken>
+  return {
+    ...hook,
+    isContainingBlock: false,
+  }
+}

package/src/parse.ts

@@ -0,0 +1,58 @@
+import { CodeType } from '@yozora/ast'
+import type { INodePoint } from '@yozora/character'
+import {
+  calcEscapedStringFromNodePoints,
+  calcStringFromNodePoints,
+  isUnicodeWhitespaceCharacter,
+} from '@yozora/character'
+import type { IParseBlockHookCreator } from '@yozora/core-tokenizer'
+import { eatOptionalWhitespaces, mergeContentLinesFaithfully } from '@yozora/core-tokenizer'
+import type { INode, IThis, IToken, T } from './types'
+
+export const parse: IParseBlockHookCreator<T, IToken, INode, IThis> = function (api) {
+  return {
+    parse: tokens =>
+      tokens.map(token => {
+        const infoString = token.infoString
+
+        // match lang
+        let i = 0
+        const langInfo: INodePoint[] = []
+        for (; i < infoString.length; ++i) {
+          const p = infoString[i]
+          if (isUnicodeWhitespaceCharacter(p.codePoint)) break
+          langInfo.push(p)
+        }
+        const lang: string = calcEscapedStringFromNodePoints(langInfo, 0, langInfo.length, true)
+
+        // match meta
+        i = eatOptionalWhitespaces(infoString, i, infoString.length)
+        const meta: string = calcEscapedStringFromNodePoints(infoString, i, infoString.length, true)
+
+        /**
+         * match content
+         * Backslash escape works in info strings in fenced code blocks.
+         * @see https://github.github.com/gfm/#example-320
+         */
+        const contents: INodePoint[] = mergeContentLinesFaithfully(token.lines)
+        let value: string = calcStringFromNodePoints(contents)
+        if (!/\n$/.test(value)) value += '\n'
+
+        const node: INode = api.shouldReservePosition
+          ? {
+              type: CodeType,
+              position: token.position,
+              lang: lang.length > 0 ? lang : null,
+              meta: meta.length > 0 ? meta : null,
+              value,
+            }
+          : {
+              type: CodeType,
+              lang: lang.length > 0 ? lang : null,
+              meta: meta.length > 0 ? meta : null,
+              value,
+            }
+        return node
+      }),
+  }
+}

package/src/tokenizer.ts

@@ -0,0 +1,52 @@
+import { CodeType } from '@yozora/ast'
+import { AsciiCodePoint } from '@yozora/character'
+import type {
+  IBlockTokenizer,
+  IMatchBlockHookCreator,
+  IParseBlockHookCreator,
+} from '@yozora/core-tokenizer'
+import { TokenizerPriority } from '@yozora/core-tokenizer'
+import FencedBlockTokenizer from '@yozora/tokenizer-fenced-block'
+import { match } from './match'
+import { parse } from './parse'
+import type { INode, IThis, IToken, ITokenizerProps, T } from './types'
+import { uniqueName } from './types'
+
+/**
+ * Lexical Analyzer for FencedCode.
+ * @see https://github.com/syntax-tree/mdast#code
+ * @see https://github.github.com/gfm/#code-fence
+ */
+export class FencedCodeTokenizer
+  extends FencedBlockTokenizer<T, INode, IThis>
+  implements IBlockTokenizer<T, IToken, INode, IThis>
+{
+  /* istanbul ignore next */
+  constructor(props: ITokenizerProps = {}) {
+    super({
+      name: props.name ?? uniqueName,
+      priority: props.priority ?? TokenizerPriority.FENCED_BLOCK,
+      nodeType: CodeType,
+      markers: [AsciiCodePoint.BACKTICK, AsciiCodePoint.TILDE],
+      markersRequired: 3,
+      checkInfoString: (infoString, marker): boolean => {
+        /**
+         * Info strings for backtick code blocks cannot contain backticks:
+         * Info strings for tilde code blocks can contain backticks and tildes
+         * @see https://github.github.com/gfm/#example-115
+         * @see https://github.github.com/gfm/#example-116
+         */
+        if (marker === AsciiCodePoint.BACKTICK) {
+          for (const p of infoString) {
+            if (p.codePoint === AsciiCodePoint.BACKTICK) return false
+          }
+        }
+        return true
+      },
+    })
+  }
+
+  public override readonly match: IMatchBlockHookCreator<T, IToken, IThis> = match
+
+  public override readonly parse: IParseBlockHookCreator<T, IToken, INode, IThis> = parse
+}

package/src/types.ts

@@ -0,0 +1,13 @@
+import type { Code, CodeType } from '@yozora/ast'
+import type { IBaseBlockTokenizerProps } from '@yozora/core-tokenizer'
+import type { IFencedBlockHookContext, IFencedBlockToken } from '@yozora/tokenizer-fenced-block'
+
+export type T = CodeType
+export type INode = Code
+export const uniqueName = '@yozora/tokenizer-fenced-code'
+
+export type IToken = IFencedBlockToken<T>
+
+export type IThis = IFencedBlockHookContext<T>
+
+export type ITokenizerProps = Partial<IBaseBlockTokenizerProps>