@yozora/tokenizer-inline-math 1.2.2 → 2.0.0-alpha.2

package/README.md

@@ -82,14 +82,14 @@ so you can use `YozoraParser` directly.
 registered in *YastParser* as a plugin-in before it can be used.
 
 ```typescript {4,9}
-import { DefaultYastParser } from '@yozora/core-parser'
+import { DefaultParser } from '@yozora/core-parser'
 import ParagraphTokenizer from '@yozora/tokenizer-paragraph'
 import TextTokenizer from '@yozora/tokenizer-text'
 import InlineMathTokenizer from '@yozora/tokenizer-inline-math'
 
-const parser = new DefaultYastParser()
-  .useBlockFallbackTokenizer(new ParagraphTokenizer())
-  .useInlineFallbackTokenizer(new TextTokenizer())
+const parser = new DefaultParser()
+  .useFallbackTokenizer(new ParagraphTokenizer())
+  .useFallbackTokenizer(new TextTokenizer())
   .useTokenizer(new InlineMathTokenizer())
 
 // parse source markdown content
@@ -228,7 +228,6 @@ Name              | Type        | Required  | Default
 [@yozora/tokenizer-link]:                     https://github.com/yozorajs/yozora/tree/main/tokenizers/link#readme
 [@yozora/tokenizer-link-reference]:           https://github.com/yozorajs/yozora/tree/main/tokenizers/link-reference#readme
 [@yozora/tokenizer-list]:                     https://github.com/yozorajs/yozora/tree/main/tokenizers/list#readme
-[@yozora/tokenizer-list-item]:                https://github.com/yozorajs/yozora/tree/main/tokenizers/list-item#readme
 [@yozora/tokenizer-math]:                     https://github.com/yozorajs/yozora/tree/main/tokenizers/math#readme
 [@yozora/tokenizer-paragraph]:                https://github.com/yozorajs/yozora/tree/main/tokenizers/paragraph#readme
 [@yozora/tokenizer-setext-heading]:           https://github.com/yozorajs/yozora/tree/main/tokenizers/setext-heading#readme
@@ -288,7 +287,6 @@ Name              | Type        | Required  | Default
 [doc-@yozora/tokenizer-definition]:           https://yozora.guanghechen.com/docs/package/tokenizer-definition
 [doc-@yozora/tokenizer-link-reference]:       https://yozora.guanghechen.com/docs/package/tokenizer-link-reference
 [doc-@yozora/tokenizer-list]:                 https://yozora.guanghechen.com/docs/package/tokenizer-list
-[doc-@yozora/tokenizer-list-item]:            https://yozora.guanghechen.com/docs/package/tokenizer-list-item
 [doc-@yozora/tokenizer-math]:                 https://yozora.guanghechen.com/docs/package/tokenizer-math
 [doc-@yozora/tokenizer-paragraph]:            https://yozora.guanghechen.com/docs/package/tokenizer-paragraph
 [doc-@yozora/tokenizer-setext-heading]:       https://yozora.guanghechen.com/docs/package/tokenizer-setext-heading

package/lib/cjs/index.js

@@ -6,18 +6,11 @@ var ast = require('@yozora/ast');
 var character = require('@yozora/character');
 var coreTokenizer = require('@yozora/core-tokenizer');
 
-const uniqueName = '@yozora/tokenizer-inline-math';
-
-class InlineMathTokenizer extends coreTokenizer.BaseInlineTokenizer {
-    constructor(props = {}) {
-        var _a, _b, _c;
-        super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.ATOMIC,
-        });
-        this.backtickRequired = (_c = props.backtickRequired) !== null && _c !== void 0 ? _c : true;
-    }
-    *findDelimiter(nodePoints, api) {
+const match = function (api) {
+    const { backtickRequired } = this;
+    return { findDelimiter, processSingleDelimiter };
+    function* findDelimiter() {
+        const nodePoints = api.getNodePoints();
         const blockStartIndex = api.getBlockStartIndex();
         const blockEndIndex = api.getBlockEndIndex();
         const potentialDelimiters = [];
@@ -30,8 +23,7 @@ class InlineMathTokenizer extends coreTokenizer.BaseInlineTokenizer {
                 case character.AsciiCodePoint.BACKTICK: {
                     const _startIndex = i;
                     i = coreTokenizer.eatOptionalCharacters(nodePoints, i + 1, blockEndIndex, character.AsciiCodePoint.BACKTICK);
-                    if (i >= blockEndIndex ||
-                        nodePoints[i].codePoint !== character.AsciiCodePoint.DOLLAR_SIGN) {
+                    if (i >= blockEndIndex || nodePoints[i].codePoint !== character.AsciiCodePoint.DOLLAR_SIGN) {
                         break;
                     }
                     const delimiter = {
@@ -45,13 +37,12 @@ class InlineMathTokenizer extends coreTokenizer.BaseInlineTokenizer {
                 case character.AsciiCodePoint.DOLLAR_SIGN: {
                     const _startIndex = i;
                     i = coreTokenizer.eatOptionalCharacters(nodePoints, i + 1, blockEndIndex, character.AsciiCodePoint.BACKTICK);
-                    if (i < blockEndIndex &&
-                        nodePoints[i].codePoint === character.AsciiCodePoint.DOLLAR_SIGN) {
+                    if (i < blockEndIndex && nodePoints[i].codePoint === character.AsciiCodePoint.DOLLAR_SIGN) {
                         break;
                     }
                     const thickness = i - _startIndex;
                     if (thickness <= 1) {
-                        if (this.backtickRequired)
+                        if (backtickRequired)
                             break;
                         const delimiter = {
                             type: 'both',
@@ -115,10 +106,7 @@ class InlineMathTokenizer extends coreTokenizer.BaseInlineTokenizer {
             };
         }
     }
-    _findDelimiter() {
-        return null;
-    }
-    processSingleDelimiter(delimiter) {
+    function processSingleDelimiter(delimiter) {
         const token = {
             nodeType: ast.InlineMathType,
             startIndex: delimiter.startIndex,
@@ -127,32 +115,55 @@ class InlineMathTokenizer extends coreTokenizer.BaseInlineTokenizer {
         };
         return [token];
     }
-    processToken(token, children, nodePoints) {
-        let startIndex = token.startIndex + token.thickness;
-        let endIndex = token.endIndex - token.thickness;
-        let isAllSpace = true;
-        for (let i = startIndex; i < endIndex; ++i) {
-            if (character.isSpaceLike(nodePoints[i].codePoint))
-                continue;
-            isAllSpace = false;
-            break;
-        }
-        if (!isAllSpace && startIndex + 2 < endIndex) {
-            const firstCharacter = nodePoints[startIndex].codePoint;
-            const lastCharacter = nodePoints[endIndex - 1].codePoint;
-            if (character.isSpaceLike(firstCharacter) && character.isSpaceLike(lastCharacter)) {
-                startIndex += 1;
-                endIndex -= 1;
+};
+
+const parse = function (api) {
+    return {
+        parse: tokens => tokens.map(token => {
+            const nodePoints = api.getNodePoints();
+            let startIndex = token.startIndex + token.thickness;
+            let endIndex = token.endIndex - token.thickness;
+            let isAllSpace = true;
+            for (let i = startIndex; i < endIndex; ++i) {
+                if (character.isSpaceLike(nodePoints[i].codePoint))
+                    continue;
+                isAllSpace = false;
+                break;
             }
-        }
-        const result = {
-            type: ast.InlineMathType,
-            value: character.calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/, ' '),
-        };
-        return result;
+            if (!isAllSpace && startIndex + 2 < endIndex) {
+                const firstCharacter = nodePoints[startIndex].codePoint;
+                const lastCharacter = nodePoints[endIndex - 1].codePoint;
+                if (character.isSpaceLike(firstCharacter) && character.isSpaceLike(lastCharacter)) {
+                    startIndex += 1;
+                    endIndex -= 1;
+                }
+            }
+            const value = character.calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/, ' ');
+            const node = api.shouldReservePosition
+                ? { type: ast.InlineMathType, position: api.calcPosition(token), value }
+                : { type: ast.InlineMathType, value };
+            return node;
+        }),
+    };
+};
+
+const uniqueName = '@yozora/tokenizer-inline-math';
+
+class InlineMathTokenizer extends coreTokenizer.BaseInlineTokenizer {
+    constructor(props = {}) {
+        var _a, _b, _c;
+        super({
+            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
+            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.ATOMIC,
+        });
+        this.match = match;
+        this.parse = parse;
+        this.backtickRequired = (_c = props.backtickRequired) !== null && _c !== void 0 ? _c : true;
     }
 }
 
 exports.InlineMathTokenizer = InlineMathTokenizer;
 exports.InlineMathTokenizerName = uniqueName;
-exports['default'] = InlineMathTokenizer;
+exports["default"] = InlineMathTokenizer;
+exports.inlineMathMatch = match;
+exports.inlineMathParse = parse;

package/lib/esm/index.js

@@ -1,19 +1,12 @@
 import { InlineMathType } from '@yozora/ast';
 import { AsciiCodePoint, isSpaceLike, calcStringFromNodePoints } from '@yozora/character';
-import { BaseInlineTokenizer, TokenizerPriority, eatOptionalCharacters } from '@yozora/core-tokenizer';
+import { eatOptionalCharacters, BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
 
-const uniqueName = '@yozora/tokenizer-inline-math';
-
-class InlineMathTokenizer extends BaseInlineTokenizer {
-    constructor(props = {}) {
-        var _a, _b, _c;
-        super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
-        });
-        this.backtickRequired = (_c = props.backtickRequired) !== null && _c !== void 0 ? _c : true;
-    }
-    *findDelimiter(nodePoints, api) {
+const match = function (api) {
+    const { backtickRequired } = this;
+    return { findDelimiter, processSingleDelimiter };
+    function* findDelimiter() {
+        const nodePoints = api.getNodePoints();
         const blockStartIndex = api.getBlockStartIndex();
         const blockEndIndex = api.getBlockEndIndex();
         const potentialDelimiters = [];
@@ -26,8 +19,7 @@ class InlineMathTokenizer extends BaseInlineTokenizer {
                 case AsciiCodePoint.BACKTICK: {
                     const _startIndex = i;
                     i = eatOptionalCharacters(nodePoints, i + 1, blockEndIndex, AsciiCodePoint.BACKTICK);
-                    if (i >= blockEndIndex ||
-                        nodePoints[i].codePoint !== AsciiCodePoint.DOLLAR_SIGN) {
+                    if (i >= blockEndIndex || nodePoints[i].codePoint !== AsciiCodePoint.DOLLAR_SIGN) {
                         break;
                     }
                     const delimiter = {
@@ -41,13 +33,12 @@ class InlineMathTokenizer extends BaseInlineTokenizer {
                 case AsciiCodePoint.DOLLAR_SIGN: {
                     const _startIndex = i;
                     i = eatOptionalCharacters(nodePoints, i + 1, blockEndIndex, AsciiCodePoint.BACKTICK);
-                    if (i < blockEndIndex &&
-                        nodePoints[i].codePoint === AsciiCodePoint.DOLLAR_SIGN) {
+                    if (i < blockEndIndex && nodePoints[i].codePoint === AsciiCodePoint.DOLLAR_SIGN) {
                         break;
                     }
                     const thickness = i - _startIndex;
                     if (thickness <= 1) {
-                        if (this.backtickRequired)
+                        if (backtickRequired)
                             break;
                         const delimiter = {
                             type: 'both',
@@ -111,10 +102,7 @@ class InlineMathTokenizer extends BaseInlineTokenizer {
             };
         }
     }
-    _findDelimiter() {
-        return null;
-    }
-    processSingleDelimiter(delimiter) {
+    function processSingleDelimiter(delimiter) {
         const token = {
             nodeType: InlineMathType,
             startIndex: delimiter.startIndex,
@@ -123,30 +111,51 @@ class InlineMathTokenizer extends BaseInlineTokenizer {
         };
         return [token];
     }
-    processToken(token, children, nodePoints) {
-        let startIndex = token.startIndex + token.thickness;
-        let endIndex = token.endIndex - token.thickness;
-        let isAllSpace = true;
-        for (let i = startIndex; i < endIndex; ++i) {
-            if (isSpaceLike(nodePoints[i].codePoint))
-                continue;
-            isAllSpace = false;
-            break;
-        }
-        if (!isAllSpace && startIndex + 2 < endIndex) {
-            const firstCharacter = nodePoints[startIndex].codePoint;
-            const lastCharacter = nodePoints[endIndex - 1].codePoint;
-            if (isSpaceLike(firstCharacter) && isSpaceLike(lastCharacter)) {
-                startIndex += 1;
-                endIndex -= 1;
+};
+
+const parse = function (api) {
+    return {
+        parse: tokens => tokens.map(token => {
+            const nodePoints = api.getNodePoints();
+            let startIndex = token.startIndex + token.thickness;
+            let endIndex = token.endIndex - token.thickness;
+            let isAllSpace = true;
+            for (let i = startIndex; i < endIndex; ++i) {
+                if (isSpaceLike(nodePoints[i].codePoint))
+                    continue;
+                isAllSpace = false;
+                break;
             }
-        }
-        const result = {
-            type: InlineMathType,
-            value: calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/, ' '),
-        };
-        return result;
+            if (!isAllSpace && startIndex + 2 < endIndex) {
+                const firstCharacter = nodePoints[startIndex].codePoint;
+                const lastCharacter = nodePoints[endIndex - 1].codePoint;
+                if (isSpaceLike(firstCharacter) && isSpaceLike(lastCharacter)) {
+                    startIndex += 1;
+                    endIndex -= 1;
+                }
+            }
+            const value = calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/, ' ');
+            const node = api.shouldReservePosition
+                ? { type: InlineMathType, position: api.calcPosition(token), value }
+                : { type: InlineMathType, value };
+            return node;
+        }),
+    };
+};
+
+const uniqueName = '@yozora/tokenizer-inline-math';
+
+class InlineMathTokenizer extends BaseInlineTokenizer {
+    constructor(props = {}) {
+        var _a, _b, _c;
+        super({
+            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
+            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
+        });
+        this.match = match;
+        this.parse = parse;
+        this.backtickRequired = (_c = props.backtickRequired) !== null && _c !== void 0 ? _c : true;
     }
 }
 
-export { InlineMathTokenizer, uniqueName as InlineMathTokenizerName, InlineMathTokenizer as default };
+export { InlineMathTokenizer, uniqueName as InlineMathTokenizerName, InlineMathTokenizer as default, match as inlineMathMatch, parse as inlineMathParse };

package/lib/types/index.d.ts

@@ -1,5 +1,5 @@
-import { InlineMathTokenizer } from './tokenizer';
-export { InlineMathTokenizer } from './tokenizer';
+export { match as inlineMathMatch } from './match';
+export { parse as inlineMathParse } from './parse';
+export { InlineMathTokenizer, InlineMathTokenizer as default } from './tokenizer';
 export { uniqueName as InlineMathTokenizerName } from './types';
-export type { Token as InlineMathToken, TokenizerProps as InlineMathTokenizerProps, } from './types';
-export default InlineMathTokenizer;
+export type { IThis as IInlineMathHookContext, IToken as IInlineMathToken, ITokenizerProps as IInlineMathTokenizerProps, } from './types';

package/lib/types/match.d.ts

@@ -0,0 +1,3 @@
+import type { IMatchInlineHookCreator } from '@yozora/core-tokenizer';
+import type { IDelimiter, IThis, IToken, T } from './types';
+export declare const match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis>;

package/lib/types/parse.d.ts

@@ -0,0 +1,3 @@
+import type { IParseInlineHookCreator } from '@yozora/core-tokenizer';
+import type { INode, IThis, IToken, T } from './types';
+export declare const parse: IParseInlineHookCreator<T, IToken, INode, IThis>;

package/lib/types/tokenizer.d.ts

@@ -1,32 +1,12 @@
-import type { YastNode } from '@yozora/ast';
-import type { NodePoint } from '@yozora/character';
-import type { MatchInlinePhaseApi, ResultOfFindDelimiters, ResultOfProcessSingleDelimiter, Tokenizer, TokenizerMatchInlineHook, TokenizerParseInlineHook } from '@yozora/core-tokenizer';
+import type { IInlineTokenizer, IMatchInlineHookCreator, IParseInlineHookCreator } from '@yozora/core-tokenizer';
 import { BaseInlineTokenizer } from '@yozora/core-tokenizer';
-import type { Delimiter, Node, T, Token, TokenizerProps } from './types';
+import type { IDelimiter, INode, IThis, IToken, ITokenizerProps, T } from './types';
 /**
  * Lexical Analyzer for inlineMath.
  */
-export declare class InlineMathTokenizer extends BaseInlineTokenizer<Delimiter> implements Tokenizer, TokenizerMatchInlineHook<T, Delimiter, Token>, TokenizerParseInlineHook<T, Token, Node> {
+export declare class InlineMathTokenizer extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis> implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis> {
     readonly backtickRequired: boolean;
-    constructor(props?: TokenizerProps);
-    /**
-     * @override
-     * @see BaseInlineTokenizer
-     */
-    findDelimiter(nodePoints: ReadonlyArray<NodePoint>, api: Readonly<MatchInlinePhaseApi>): ResultOfFindDelimiters<Delimiter>;
-    /**
-     * @override
-     * @see BaseInlineTokenizer
-     */
-    protected _findDelimiter(): null;
-    /**
-     * @override
-     * @see TokenizerMatchInlineHook
-     */
-    processSingleDelimiter(delimiter: Delimiter): ResultOfProcessSingleDelimiter<T, Token>;
-    /**
-     * @override
-     * @see TokenizerParseInlineHook
-     */
-    processToken(token: Token, children: YastNode[] | undefined, nodePoints: ReadonlyArray<NodePoint>): Node;
+    constructor(props?: ITokenizerProps);
+    readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis>;
+    readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis>;
 }

package/lib/types/types.d.ts

@@ -1,25 +1,31 @@
-import type { InlineMath, InlineMathType } from '@yozora/ast';
-import type { BaseInlineTokenizerProps, PartialYastInlineToken, YastTokenDelimiter } from '@yozora/core-tokenizer';
+import type { IInlineMath, InlineMathType } from '@yozora/ast';
+import type { IBaseInlineTokenizerProps, IPartialYastInlineToken, ITokenizer, IYastTokenDelimiter } from '@yozora/core-tokenizer';
 export declare type T = InlineMathType;
-export declare type Node = InlineMath;
+export declare type INode = IInlineMath;
 export declare const uniqueName = "@yozora/tokenizer-inline-math";
-export interface Token extends PartialYastInlineToken<T> {
+export interface IToken extends IPartialYastInlineToken<T> {
     /**
      * Thickness of the InlineMathDelimiter
      */
     thickness: number;
 }
 /**
- * Delimiter of InlineMathToken.
+ * IDelimiter of InlineMathToken.
  */
-export interface Delimiter extends YastTokenDelimiter {
+export interface IDelimiter extends IYastTokenDelimiter {
     type: 'full';
     /**
      * Thickness of the InlineMathDelimiter
      */
     thickness: number;
 }
-export interface TokenizerProps extends Partial<BaseInlineTokenizerProps> {
+export interface IThis extends ITokenizer {
+    /**
+     * Whether if the backtick mark wrapping necessary.
+     */
+    readonly backtickRequired: boolean;
+}
+export interface ITokenizerProps extends Partial<IBaseInlineTokenizerProps> {
     /**
      * Whether if the backtick mark wrapping necessary.
      * @default true

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-inline-math",
-  "version": "1.2.2",
+  "version": "2.0.0-alpha.2",
   "description": "Tokenizer for processing inline math (formulas)",
   "author": {
     "name": "guanghechen",
@@ -45,12 +45,12 @@
     "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^1.2.2",
-    "@yozora/character": "^1.2.2",
-    "@yozora/core-tokenizer": "^1.2.2"
+    "@yozora/ast": "^2.0.0-alpha.2",
+    "@yozora/character": "^2.0.0-alpha.2",
+    "@yozora/core-tokenizer": "^2.0.0-alpha.2"
   },
   "devDependencies": {
-    "@yozora/tokenizer-inline-code": "^1.2.2"
+    "@yozora/tokenizer-inline-code": "^2.0.0-alpha.2"
   },
-  "gitHead": "fb4b804548592c09259aabcf9cc4dd96176bd024"
+  "gitHead": "da59d85520455c59a117a35032ef1a035c10ea21"
 }