@yozora/tokenizer-break 2.0.3 → 2.0.5-alpha.0

package/lib/cjs/{index.js → index.cjs}

@@ -86,18 +86,17 @@ const parse = function (api) {
 
 class BreakTokenizer extends coreTokenizer.BaseInlineTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.SOFT_INLINE,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? coreTokenizer.TokenizerPriority.SOFT_INLINE,
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 exports.BreakTokenizer = BreakTokenizer;
 exports.BreakTokenizerName = uniqueName;
 exports.breakMatch = match;
 exports.breakParse = parse;
-exports["default"] = BreakTokenizer;
+exports.default = BreakTokenizer;

package/lib/esm/{index.js → index.mjs}

@@ -82,14 +82,13 @@ const parse = function (api) {
 
 class BreakTokenizer extends BaseInlineTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.SOFT_INLINE,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? TokenizerPriority.SOFT_INLINE,
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 export { BreakTokenizer, uniqueName as BreakTokenizerName, match as breakMatch, parse as breakParse, BreakTokenizer as default };

package/lib/types/index.d.ts

@@ -1,10 +1,10 @@
 import { IPartialYastInlineToken, ITokenizer, IBaseInlineTokenizerProps, IYastTokenDelimiter, IMatchInlineHookCreator, IParseInlineHookCreator, BaseInlineTokenizer, IInlineTokenizer } from '@yozora/core-tokenizer';
 import { BreakType, Break } from '@yozora/ast';
 
-declare type T = BreakType;
-declare type INode = Break;
+type T = BreakType;
+type INode = Break;
 declare const uniqueName = "@yozora/tokenizer-break";
-declare type IToken = IPartialYastInlineToken<T>;
+type IToken = IPartialYastInlineToken<T>;
 interface IDelimiter extends IYastTokenDelimiter {
     type: 'full';
     /**
@@ -12,8 +12,8 @@ interface IDelimiter extends IYastTokenDelimiter {
      */
     markerType: BreakTokenMarkerType;
 }
-declare type IThis = ITokenizer;
-declare type ITokenizerProps = Partial<IBaseInlineTokenizerProps>;
+type IThis = ITokenizer;
+type ITokenizerProps = Partial<IBaseInlineTokenizerProps>;
 /**
  * Line break marker type.
  */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-break",
-  "version": "2.0.3",
+  "version": "2.0.5-alpha.0",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -11,33 +11,37 @@
     "directory": "tokenizers/break"
   },
   "homepage": "https://github.com/yozorajs/yozora/tree/release-2.x.x/tokenizers/break",
-  "main": "lib/cjs/index.js",
-  "module": "lib/esm/index.js",
-  "types": "lib/types/index.d.ts",
-  "source": "src/index.ts",
+  "type": "module",
+  "exports": {
+    "types": "./lib/types/index.d.ts",
+    "import": "./lib/esm/index.mjs",
+    "require": "./lib/cjs/index.cjs"
+  },
+  "source": "./src/index.ts",
+  "types": "./lib/types/index.d.ts",
+  "main": "./lib/cjs/index.cjs",
+  "module": "./lib/esm/index.mjs",
   "license": "MIT",
   "engines": {
     "node": ">= 16.0.0"
   },
   "files": [
     "lib/",
-    "!lib/**/*.js.map",
-    "!lib/**/*.d.ts.map",
+    "src/",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
     "README.md"
   ],
   "scripts": {
-    "build": "cross-env NODE_ENV=production rollup -c ../../rollup.config.js",
-    "prebuild": "rimraf lib/",
+    "build": "rimraf lib/ && cross-env NODE_ENV=production rollup -c ../../rollup.config.mjs",
     "prepublishOnly": "cross-env ROLLUP_SHOULD_SOURCEMAP=false yarn build",
-    "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
+    "test": "cross-env TS_NODE_FILES=true NODE_OPTIONS=--experimental-vm-modules jest --config ../../jest.config.mjs --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.0.3",
-    "@yozora/character": "^2.0.3",
-    "@yozora/core-tokenizer": "^2.0.3"
+    "@yozora/ast": "^2.0.5-alpha.0",
+    "@yozora/character": "^2.0.5-alpha.0",
+    "@yozora/core-tokenizer": "^2.0.5-alpha.0"
   },
-  "gitHead": "8cc8f95cfebc8d752bc3272cdd24965f540c130b"
+  "gitHead": "8bf941fe4ef82947165b0f3cc123cd493665e13b"
 }

package/src/index.ts

@@ -0,0 +1,9 @@
+export { match as breakMatch } from './match'
+export { parse as breakParse } from './parse'
+export { BreakTokenizer, BreakTokenizer as default } from './tokenizer'
+export { uniqueName as BreakTokenizerName } from './types'
+export type {
+  IThis as IBreakHookContext,
+  IToken as IBreakToken,
+  ITokenizerProps as IBreakTokenizerProps,
+} from './types'

package/src/match.ts

@@ -0,0 +1,104 @@
+import { BreakType } from '@yozora/ast'
+import type { INodePoint } from '@yozora/character'
+import { AsciiCodePoint, VirtualCodePoint } from '@yozora/character'
+import type {
+  IMatchInlineHookCreator,
+  IResultOfProcessSingleDelimiter,
+} from '@yozora/core-tokenizer'
+import { genFindDelimiter } from '@yozora/core-tokenizer'
+import { BreakTokenMarkerType } from './types'
+import type { IDelimiter, IThis, IToken, T } from './types'
+
+/**
+ * A line break (not in a code span or HTML tag) that is preceded by two or more
+ * spaces and does not occur at the end of a block is parsed as a hard line
+ * break (rendered in HTML as a <br /> tag)
+ * @see https://github.github.com/gfm/#hard-line-breaks
+ *
+ * A regular line break (not in a code span or HTML tag) that is not preceded
+ * by two or more spaces or a backslash is parsed as a softbreak. (A softbreak
+ * may be rendered in HTML either as a line ending or as a space. The result
+ * will be the same in browsers.
+ * @see https://github.github.com/gfm/#soft-line-breaks
+ *
+ * @see https://github.com/syntax-tree/mdast#break
+ */
+export const match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = function (api) {
+  return {
+    findDelimiter: () => genFindDelimiter<IDelimiter>(_findDelimiter),
+    processSingleDelimiter,
+  }
+
+  function _findDelimiter(startIndex: number, endIndex: number): IDelimiter | null {
+    const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
+    for (let i = startIndex + 1; i < endIndex; ++i) {
+      if (nodePoints[i].codePoint !== VirtualCodePoint.LINE_END) continue
+
+      const c = nodePoints[i - 1].codePoint
+      let _start: number | null = null
+      let markerType: BreakTokenMarkerType | null = null
+      switch (c) {
+        /**
+         * For a more visible alternative, a backslash
+         * before the line ending may be used instead of two spaces
+         * @see https://github.github.com/gfm/#example-655
+         */
+        case AsciiCodePoint.BACKSLASH: {
+          let x = i - 2
+          for (; x >= startIndex; x -= 1) {
+            if (nodePoints[x].codePoint !== AsciiCodePoint.BACKSLASH) break
+          }
+          if (((i - x) & 1) === 0) {
+            _start = i - 1
+            markerType = BreakTokenMarkerType.BACKSLASH
+          }
+          break
+        }
+        /**
+         * - A line break (not in a code span or HTML tag) that is preceded
+         *   by two or more spaces and does not occur at the end of a block
+         *   is parsed as a hard line break (rendered in HTML as a <br /> tag)
+         * - More than two spaces can be used
+         * - Leading spaces at the beginning of the next line are ignored
+         *
+         * @see https://github.github.com/gfm/#example-654
+         * @see https://github.github.com/gfm/#example-656
+         * @see https://github.github.com/gfm/#example-657
+         */
+        case AsciiCodePoint.SPACE: {
+          let x = i - 2
+          for (; x >= startIndex; x -= 1) {
+            if (nodePoints[x].codePoint !== AsciiCodePoint.SPACE) break
+          }
+
+          if (i - x > 2) {
+            _start = x + 1
+            markerType = BreakTokenMarkerType.MORE_THAN_TWO_SPACES
+          }
+          break
+        }
+      }
+
+      if (_start == null || markerType == null) continue
+
+      return {
+        type: 'full',
+        markerType,
+        startIndex: _start,
+        endIndex: i,
+      }
+    }
+    return null
+  }
+
+  function processSingleDelimiter(
+    delimiter: IDelimiter,
+  ): IResultOfProcessSingleDelimiter<T, IToken> {
+    const token: IToken = {
+      nodeType: BreakType,
+      startIndex: delimiter.startIndex,
+      endIndex: delimiter.endIndex,
+    }
+    return [token]
+  }
+}

package/src/parse.ts

@@ -0,0 +1,15 @@
+import { BreakType } from '@yozora/ast'
+import type { IParseInlineHookCreator } from '@yozora/core-tokenizer'
+import type { INode, IThis, IToken, T } from './types'
+
+export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function (api) {
+  return {
+    parse: tokens =>
+      tokens.map(token => {
+        const node: INode = api.shouldReservePosition
+          ? { type: BreakType, position: api.calcPosition(token) }
+          : { type: BreakType }
+        return node
+      }),
+  }
+}

package/src/tokenizer.ts

@@ -0,0 +1,33 @@
+import type {
+  IInlineTokenizer,
+  IMatchInlineHookCreator,
+  IParseInlineHookCreator,
+} from '@yozora/core-tokenizer'
+import { BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer'
+import { match } from './match'
+import { parse } from './parse'
+import { uniqueName } from './types'
+import type { IDelimiter, INode, IThis, IToken, ITokenizerProps, T } from './types'
+
+/**
+ * Lexical Analyzer for a line break.
+ * @see https://github.github.com/gfm/#hard-line-breaks
+ * @see https://github.github.com/gfm/#soft-line-breaks
+ * @see https://github.com/syntax-tree/mdast#break
+ */
+export class BreakTokenizer
+  extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
+  implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
+{
+  /* istanbul ignore next */
+  constructor(props: ITokenizerProps = {}) {
+    super({
+      name: props.name ?? uniqueName,
+      priority: props.priority ?? TokenizerPriority.SOFT_INLINE,
+    })
+  }
+
+  public override readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = match
+
+  public override readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis> = parse
+}

package/src/types.ts

@@ -0,0 +1,39 @@
+import type { Break, BreakType } from '@yozora/ast'
+import type {
+  IBaseInlineTokenizerProps,
+  IPartialYastInlineToken,
+  ITokenizer,
+  IYastTokenDelimiter,
+} from '@yozora/core-tokenizer'
+
+export type T = BreakType
+export type INode = Break
+export const uniqueName = '@yozora/tokenizer-break'
+
+export type IToken = IPartialYastInlineToken<T>
+
+export interface IDelimiter extends IYastTokenDelimiter {
+  type: 'full'
+  /**
+   * Line break marker type.
+   */
+  markerType: BreakTokenMarkerType
+}
+
+export type IThis = ITokenizer
+
+export type ITokenizerProps = Partial<IBaseInlineTokenizerProps>
+
+/**
+ * Line break marker type.
+ */
+export enum BreakTokenMarkerType {
+  /**
+   * Backslash at the end of the line
+   */
+  BACKSLASH = 'backslash',
+  /**
+   * More than two spaces at the end of the line
+   */
+  MORE_THAN_TWO_SPACES = 'more-than-two-spaces',
+}