@yozora/tokenizer-break 2.1.2 → 2.1.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-break",
-  "version": "2.1.2",
+  "version": "2.1.4",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -27,7 +27,7 @@
   },
   "files": [
     "lib/",
-    "src/",
+    "lib/**/*.map",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
@@ -39,9 +39,9 @@
     "test": "cross-env TS_NODE_FILES=true NODE_OPTIONS=--experimental-vm-modules jest --config ../../jest.config.mjs --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.1.2",
-    "@yozora/character": "^2.1.2",
-    "@yozora/core-tokenizer": "^2.1.2"
+    "@yozora/ast": "^2.1.4",
+    "@yozora/character": "^2.1.4",
+    "@yozora/core-tokenizer": "^2.1.4"
   },
-  "gitHead": "992bacafd173e7788e99fed34ce8b45f6ed24cfe"
+  "gitHead": "aa464ed1e3cd84892773a833910cfc53a556bf5f"
 }

package/src/index.ts

@@ -1,9 +0,0 @@
-export { match as breakMatch } from './match'
-export { parse as breakParse } from './parse'
-export { BreakTokenizer, BreakTokenizer as default } from './tokenizer'
-export { uniqueName as BreakTokenizerName } from './types'
-export type {
-  IThis as IBreakHookContext,
-  IToken as IBreakToken,
-  ITokenizerProps as IBreakTokenizerProps,
-} from './types'

package/src/match.ts

@@ -1,104 +0,0 @@
-import { BreakType } from '@yozora/ast'
-import type { INodePoint } from '@yozora/character'
-import { AsciiCodePoint, VirtualCodePoint } from '@yozora/character'
-import type {
-  IMatchInlineHookCreator,
-  IResultOfProcessSingleDelimiter,
-} from '@yozora/core-tokenizer'
-import { genFindDelimiter } from '@yozora/core-tokenizer'
-import { BreakTokenMarkerType } from './types'
-import type { IDelimiter, IThis, IToken, T } from './types'
-
-/**
- * A line break (not in a code span or HTML tag) that is preceded by two or more
- * spaces and does not occur at the end of a block is parsed as a hard line
- * break (rendered in HTML as a <br /> tag)
- * @see https://github.github.com/gfm/#hard-line-breaks
- *
- * A regular line break (not in a code span or HTML tag) that is not preceded
- * by two or more spaces or a backslash is parsed as a softbreak. (A softbreak
- * may be rendered in HTML either as a line ending or as a space. The result
- * will be the same in browsers.
- * @see https://github.github.com/gfm/#soft-line-breaks
- *
- * @see https://github.com/syntax-tree/mdast#break
- */
-export const match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = function (api) {
-  return {
-    findDelimiter: () => genFindDelimiter<IDelimiter>(_findDelimiter),
-    processSingleDelimiter,
-  }
-
-  function _findDelimiter(startIndex: number, endIndex: number): IDelimiter | null {
-    const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
-    for (let i = startIndex + 1; i < endIndex; ++i) {
-      if (nodePoints[i].codePoint !== VirtualCodePoint.LINE_END) continue
-
-      const c = nodePoints[i - 1].codePoint
-      let _start: number | null = null
-      let markerType: BreakTokenMarkerType | null = null
-      switch (c) {
-        /**
-         * For a more visible alternative, a backslash
-         * before the line ending may be used instead of two spaces
-         * @see https://github.github.com/gfm/#example-655
-         */
-        case AsciiCodePoint.BACKSLASH: {
-          let x = i - 2
-          for (; x >= startIndex; x -= 1) {
-            if (nodePoints[x].codePoint !== AsciiCodePoint.BACKSLASH) break
-          }
-          if (((i - x) & 1) === 0) {
-            _start = i - 1
-            markerType = BreakTokenMarkerType.BACKSLASH
-          }
-          break
-        }
-        /**
-         * - A line break (not in a code span or HTML tag) that is preceded
-         *   by two or more spaces and does not occur at the end of a block
-         *   is parsed as a hard line break (rendered in HTML as a <br /> tag)
-         * - More than two spaces can be used
-         * - Leading spaces at the beginning of the next line are ignored
-         *
-         * @see https://github.github.com/gfm/#example-654
-         * @see https://github.github.com/gfm/#example-656
-         * @see https://github.github.com/gfm/#example-657
-         */
-        case AsciiCodePoint.SPACE: {
-          let x = i - 2
-          for (; x >= startIndex; x -= 1) {
-            if (nodePoints[x].codePoint !== AsciiCodePoint.SPACE) break
-          }
-
-          if (i - x > 2) {
-            _start = x + 1
-            markerType = BreakTokenMarkerType.MORE_THAN_TWO_SPACES
-          }
-          break
-        }
-      }
-
-      if (_start == null || markerType == null) continue
-
-      return {
-        type: 'full',
-        markerType,
-        startIndex: _start,
-        endIndex: i,
-      }
-    }
-    return null
-  }
-
-  function processSingleDelimiter(
-    delimiter: IDelimiter,
-  ): IResultOfProcessSingleDelimiter<T, IToken> {
-    const token: IToken = {
-      nodeType: BreakType,
-      startIndex: delimiter.startIndex,
-      endIndex: delimiter.endIndex,
-    }
-    return [token]
-  }
-}

package/src/parse.ts

@@ -1,15 +0,0 @@
-import { BreakType } from '@yozora/ast'
-import type { IParseInlineHookCreator } from '@yozora/core-tokenizer'
-import type { INode, IThis, IToken, T } from './types'
-
-export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function (api) {
-  return {
-    parse: tokens =>
-      tokens.map(token => {
-        const node: INode = api.shouldReservePosition
-          ? { type: BreakType, position: api.calcPosition(token) }
-          : { type: BreakType }
-        return node
-      }),
-  }
-}

package/src/tokenizer.ts

@@ -1,33 +0,0 @@
-import type {
-  IInlineTokenizer,
-  IMatchInlineHookCreator,
-  IParseInlineHookCreator,
-} from '@yozora/core-tokenizer'
-import { BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer'
-import { match } from './match'
-import { parse } from './parse'
-import { uniqueName } from './types'
-import type { IDelimiter, INode, IThis, IToken, ITokenizerProps, T } from './types'
-
-/**
- * Lexical Analyzer for a line break.
- * @see https://github.github.com/gfm/#hard-line-breaks
- * @see https://github.github.com/gfm/#soft-line-breaks
- * @see https://github.com/syntax-tree/mdast#break
- */
-export class BreakTokenizer
-  extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
-  implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
-{
-  /* istanbul ignore next */
-  constructor(props: ITokenizerProps = {}) {
-    super({
-      name: props.name ?? uniqueName,
-      priority: props.priority ?? TokenizerPriority.SOFT_INLINE,
-    })
-  }
-
-  public override readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = match
-
-  public override readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis> = parse
-}

package/src/types.ts

@@ -1,39 +0,0 @@
-import type { Break, BreakType } from '@yozora/ast'
-import type {
-  IBaseInlineTokenizerProps,
-  IPartialInlineToken,
-  ITokenDelimiter,
-  ITokenizer,
-} from '@yozora/core-tokenizer'
-
-export type T = BreakType
-export type INode = Break
-export const uniqueName = '@yozora/tokenizer-break'
-
-export type IToken = IPartialInlineToken<T>
-
-export interface IDelimiter extends ITokenDelimiter {
-  type: 'full'
-  /**
-   * Line break marker type.
-   */
-  markerType: BreakTokenMarkerType
-}
-
-export type IThis = ITokenizer
-
-export type ITokenizerProps = Partial<IBaseInlineTokenizerProps>
-
-/**
- * Line break marker type.
- */
-export enum BreakTokenMarkerType {
-  /**
-   * Backslash at the end of the line
-   */
-  BACKSLASH = 'backslash',
-  /**
-   * More than two spaces at the end of the line
-   */
-  MORE_THAN_TWO_SPACES = 'more-than-two-spaces',
-}