@yozora/tokenizer-definition 2.0.0-alpha.0 → 2.0.0

package/README.md

@@ -84,14 +84,14 @@ so you can use `YozoraParser` / `GfmExParser` / `GfmParser` directly.
 registered in *YastParser* as a plugin-in before it can be used.
 
 ```typescript {4,9}
-import { DefaultYastParser } from '@yozora/core-parser'
+import { DefaultParser } from '@yozora/core-parser'
 import ParagraphTokenizer from '@yozora/tokenizer-paragraph'
 import TextTokenizer from '@yozora/tokenizer-text'
 import DefinitionTokenizer from '@yozora/tokenizer-definition'
 
-const parser = new DefaultYastParser()
-  .useBlockFallbackTokenizer(new ParagraphTokenizer())
-  .useInlineFallbackTokenizer(new TextTokenizer())
+const parser = new DefaultParser()
+  .useFallbackTokenizer(new ParagraphTokenizer())
+  .useFallbackTokenizer(new TextTokenizer())
   .useTokenizer(new DefinitionTokenizer())
 
 // parse source markdown content
@@ -242,7 +242,6 @@ Name              | Type        | Required  | Default
 [@yozora/tokenizer-link]:                     https://github.com/yozorajs/yozora/tree/main/tokenizers/link#readme
 [@yozora/tokenizer-link-reference]:           https://github.com/yozorajs/yozora/tree/main/tokenizers/link-reference#readme
 [@yozora/tokenizer-list]:                     https://github.com/yozorajs/yozora/tree/main/tokenizers/list#readme
-[@yozora/tokenizer-list-item]:                https://github.com/yozorajs/yozora/tree/main/tokenizers/list-item#readme
 [@yozora/tokenizer-math]:                     https://github.com/yozorajs/yozora/tree/main/tokenizers/math#readme
 [@yozora/tokenizer-paragraph]:                https://github.com/yozorajs/yozora/tree/main/tokenizers/paragraph#readme
 [@yozora/tokenizer-setext-heading]:           https://github.com/yozorajs/yozora/tree/main/tokenizers/setext-heading#readme
@@ -302,7 +301,6 @@ Name              | Type        | Required  | Default
 [doc-@yozora/tokenizer-definition]:           https://yozora.guanghechen.com/docs/package/tokenizer-definition
 [doc-@yozora/tokenizer-link-reference]:       https://yozora.guanghechen.com/docs/package/tokenizer-link-reference
 [doc-@yozora/tokenizer-list]:                 https://yozora.guanghechen.com/docs/package/tokenizer-list
-[doc-@yozora/tokenizer-list-item]:            https://yozora.guanghechen.com/docs/package/tokenizer-list-item
 [doc-@yozora/tokenizer-math]:                 https://yozora.guanghechen.com/docs/package/tokenizer-math
 [doc-@yozora/tokenizer-paragraph]:            https://yozora.guanghechen.com/docs/package/tokenizer-paragraph
 [doc-@yozora/tokenizer-setext-heading]:       https://yozora.guanghechen.com/docs/package/tokenizer-setext-heading

package/lib/cjs/index.js

@@ -241,8 +241,8 @@ const match = function (api) {
             const token = {
                 nodeType: ast.DefinitionType,
                 position: {
-                    start: coreTokenizer.calcStartYastNodePoint(nodePoints, startIndex),
-                    end: coreTokenizer.calcEndYastNodePoint(nodePoints, endIndex - 1),
+                    start: coreTokenizer.calcStartPoint(nodePoints, startIndex),
+                    end: coreTokenizer.calcEndPoint(nodePoints, endIndex - 1),
                 },
                 label: labelState,
                 destination: null,
@@ -350,7 +350,7 @@ const match = function (api) {
             }
             const lastLine = token.lines[token.lines.length - 1];
             token.title = null;
-            token.position.end = coreTokenizer.calcEndYastNodePoint(lastLine.nodePoints, lastLine.endIndex - 1);
+            token.position.end = coreTokenizer.calcEndPoint(lastLine.nodePoints, lastLine.endIndex - 1);
             return {
                 status: 'closingAndRollback',
                 lines: token.lines.slice(token.lineNoOfTitle - 1),
@@ -376,7 +376,7 @@ const match = function (api) {
                 const lines = token.lines.splice(token.lineNoOfTitle - 1);
                 const lastLine = token.lines[token.lines.length - 1];
                 token.title = null;
-                token.position.end = coreTokenizer.calcEndYastNodePoint(lastLine.nodePoints, lastLine.endIndex - 1);
+                token.position.end = coreTokenizer.calcEndPoint(lastLine.nodePoints, lastLine.endIndex - 1);
                 result = { status: 'closingAndRollback', lines };
             }
         }
@@ -390,9 +390,9 @@ const match = function (api) {
     }
 };
 
-const parse = function () {
+const parse = function (api) {
     return {
-        parse: token => {
+        parse: tokens => tokens.map(token => {
             const label = token._label;
             const identifier = token._identifier;
             const destinationPoints = token.destination.nodePoints;
@@ -403,15 +403,11 @@ const parse = function () {
             const title = token.title == null
                 ? undefined
                 : character.calcEscapedStringFromNodePoints(token.title.nodePoints, 1, token.title.nodePoints.length - 1);
-            const node = {
-                type: ast.DefinitionType,
-                identifier,
-                label,
-                url,
-                title,
-            };
+            const node = api.shouldReservePosition
+                ? { type: ast.DefinitionType, position: token.position, identifier, label, url, title }
+                : { type: ast.DefinitionType, identifier, label, url, title };
             return node;
-        },
+        }),
     };
 };
 

package/lib/esm/index.js

@@ -1,5 +1,5 @@
 import { AsciiCodePoint, VirtualCodePoint, isWhitespaceCharacter, isAsciiControlCharacter, calcStringFromNodePoints, calcEscapedStringFromNodePoints } from '@yozora/character';
-import { eatOptionalWhitespaces, calcEndYastNodePoint, resolveLabelToIdentifier, calcStartYastNodePoint, encodeLinkDestination, BaseBlockTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
+import { eatOptionalWhitespaces, calcEndPoint, resolveLabelToIdentifier, calcStartPoint, encodeLinkDestination, BaseBlockTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
 import { DefinitionType } from '@yozora/ast';
 
 function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, state) {
@@ -237,8 +237,8 @@ const match = function (api) {
             const token = {
                 nodeType: DefinitionType,
                 position: {
-                    start: calcStartYastNodePoint(nodePoints, startIndex),
-                    end: calcEndYastNodePoint(nodePoints, endIndex - 1),
+                    start: calcStartPoint(nodePoints, startIndex),
+                    end: calcEndPoint(nodePoints, endIndex - 1),
                 },
                 label: labelState,
                 destination: null,
@@ -346,7 +346,7 @@ const match = function (api) {
             }
             const lastLine = token.lines[token.lines.length - 1];
             token.title = null;
-            token.position.end = calcEndYastNodePoint(lastLine.nodePoints, lastLine.endIndex - 1);
+            token.position.end = calcEndPoint(lastLine.nodePoints, lastLine.endIndex - 1);
             return {
                 status: 'closingAndRollback',
                 lines: token.lines.slice(token.lineNoOfTitle - 1),
@@ -372,7 +372,7 @@ const match = function (api) {
                 const lines = token.lines.splice(token.lineNoOfTitle - 1);
                 const lastLine = token.lines[token.lines.length - 1];
                 token.title = null;
-                token.position.end = calcEndYastNodePoint(lastLine.nodePoints, lastLine.endIndex - 1);
+                token.position.end = calcEndPoint(lastLine.nodePoints, lastLine.endIndex - 1);
                 result = { status: 'closingAndRollback', lines };
             }
         }
@@ -386,9 +386,9 @@ const match = function (api) {
     }
 };
 
-const parse = function () {
+const parse = function (api) {
     return {
-        parse: token => {
+        parse: tokens => tokens.map(token => {
             const label = token._label;
             const identifier = token._identifier;
             const destinationPoints = token.destination.nodePoints;
@@ -399,15 +399,11 @@ const parse = function () {
             const title = token.title == null
                 ? undefined
                 : calcEscapedStringFromNodePoints(token.title.nodePoints, 1, token.title.nodePoints.length - 1);
-            const node = {
-                type: DefinitionType,
-                identifier,
-                label,
-                url,
-                title,
-            };
+            const node = api.shouldReservePosition
+                ? { type: DefinitionType, position: token.position, identifier, label, url, title }
+                : { type: DefinitionType, identifier, label, url, title };
             return node;
-        },
+        }),
     };
 };
 

package/lib/types/index.d.ts

@@ -5,4 +5,4 @@ export { match as definitionMatch } from './match';
 export { parse as definitionParse } from './parse';
 export { DefinitionTokenizer, DefinitionTokenizer as default } from './tokenizer';
 export { uniqueName as DefinitionTokenizerName } from './types';
-export type { IHookContext as IDefinitionHookContext, IToken as IDefinitionToken, ITokenizerProps as IDefinitionTokenizerProps, } from './types';
+export type { IThis as IDefinitionHookContext, IToken as IDefinitionToken, ITokenizerProps as IDefinitionTokenizerProps, } from './types';

package/lib/types/match.d.ts

@@ -1,5 +1,5 @@
 import type { IMatchBlockHookCreator } from '@yozora/core-tokenizer';
-import type { IHookContext, IToken, T } from './types';
+import type { IThis, IToken, T } from './types';
 /**
  * A link reference definition consists of a link label, indented up to three
  * spaces, followed by a colon (:), optional whitespace (including up to one
@@ -15,4 +15,4 @@ import type { IHookContext, IToken, T } from './types';
  *
  * @see https://github.github.com/gfm/#link-reference-definition
  */
-export declare const match: IMatchBlockHookCreator<T, IToken, IHookContext>;
+export declare const match: IMatchBlockHookCreator<T, IToken, IThis>;

package/lib/types/parse.d.ts

@@ -1,3 +1,3 @@
 import type { IParseBlockHookCreator } from '@yozora/core-tokenizer';
-import type { IHookContext, INode, IToken, T } from './types';
-export declare const parse: IParseBlockHookCreator<T, IToken, INode, IHookContext>;
+import type { INode, IThis, IToken, T } from './types';
+export declare const parse: IParseBlockHookCreator<T, IToken, INode, IThis>;

package/lib/types/tokenizer.d.ts

@@ -1,12 +1,12 @@
 import type { IBlockTokenizer, IMatchBlockHookCreator, IParseBlockHookCreator } from '@yozora/core-tokenizer';
 import { BaseBlockTokenizer } from '@yozora/core-tokenizer';
-import type { IHookContext, INode, IToken, ITokenizerProps, T } from './types';
+import type { INode, IThis, IToken, ITokenizerProps, T } from './types';
 /**
  * Lexical Analyzer for Definition.
  * @see https://github.github.com/gfm/#link-reference-definition
  */
-export declare class DefinitionTokenizer extends BaseBlockTokenizer<T, IToken, INode, IHookContext> implements IBlockTokenizer<T, IToken, INode, IHookContext> {
+export declare class DefinitionTokenizer extends BaseBlockTokenizer<T, IToken, INode, IThis> implements IBlockTokenizer<T, IToken, INode, IThis> {
     constructor(props?: ITokenizerProps);
-    readonly match: IMatchBlockHookCreator<T, IToken, IHookContext>;
-    readonly parse: IParseBlockHookCreator<T, IToken, INode, IHookContext>;
+    readonly match: IMatchBlockHookCreator<T, IToken, IThis>;
+    readonly parse: IParseBlockHookCreator<T, IToken, INode, IThis>;
 }

package/lib/types/types.d.ts

@@ -1,10 +1,10 @@
-import type { DefinitionType, IDefinition } from '@yozora/ast';
+import type { Definition, DefinitionType } from '@yozora/ast';
 import type { IBaseBlockTokenizerProps, IPartialYastBlockToken, IPhrasingContentLine, ITokenizer } from '@yozora/core-tokenizer';
-import type { LinkDestinationCollectingState } from './util/link-destination';
-import type { LinkLabelCollectingState } from './util/link-label';
-import type { LinkTitleCollectingState } from './util/link-title';
+import type { ILinkDestinationCollectingState } from './util/link-destination';
+import type { ILinkLabelCollectingState } from './util/link-label';
+import type { ILinkTitleCollectingState } from './util/link-title';
 export declare type T = DefinitionType;
-export declare type INode = IDefinition;
+export declare type INode = Definition;
 export declare const uniqueName = "@yozora/tokenizer-definition";
 export interface IToken extends IPartialYastBlockToken<T> {
     /**
@@ -15,15 +15,15 @@ export interface IToken extends IPartialYastBlockToken<T> {
      * Link label
      * Trimmed, Case-Insensitive
      */
-    label: LinkLabelCollectingState;
+    label: ILinkLabelCollectingState;
     /**
      * Link destination
      */
-    destination: LinkDestinationCollectingState | null;
+    destination: ILinkDestinationCollectingState | null;
     /**
      * Link title
      */
-    title: LinkTitleCollectingState | null;
+    title: ILinkTitleCollectingState | null;
     /**
      * The line number of the first matched character of the link label
      */
@@ -45,5 +45,5 @@ export interface IToken extends IPartialYastBlockToken<T> {
      */
     _identifier?: string;
 }
-export declare type IHookContext = ITokenizer;
+export declare type IThis = ITokenizer;
 export declare type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;

package/lib/types/util/link-destination.d.ts

@@ -5,7 +5,7 @@ import type { INodePoint } from '@yozora/character';
  *
  * @see https://github.github.com/gfm/#link-destination
  */
-export interface LinkDestinationCollectingState {
+export interface ILinkDestinationCollectingState {
     /**
      * Whether the current token has collected a legal LinkDestination
      */
@@ -31,7 +31,7 @@ export interface LinkDestinationCollectingState {
  * @param state
  * @see https://github.github.com/gfm/#link-destination
  */
-export declare function eatAndCollectLinkDestination(nodePoints: ReadonlyArray<INodePoint>, startIndex: number, endIndex: number, state: LinkDestinationCollectingState | null): {
+export declare function eatAndCollectLinkDestination(nodePoints: ReadonlyArray<INodePoint>, startIndex: number, endIndex: number, state: ILinkDestinationCollectingState | null): {
     nextIndex: number;
-    state: LinkDestinationCollectingState;
+    state: ILinkDestinationCollectingState;
 };

package/lib/types/util/link-label.d.ts

@@ -5,7 +5,7 @@ import type { INodePoint } from '@yozora/character';
  *
  * @see https://github.github.com/gfm/#link-label
  */
-export interface LinkLabelCollectingState {
+export interface ILinkLabelCollectingState {
     /**
      * Whether the current token has collected a legal LinkDestination
      */
@@ -38,7 +38,7 @@ export interface LinkLabelCollectingState {
  * @param state
  * @see https://github.github.com/gfm/#link-label
  */
-export declare function eatAndCollectLinkLabel(nodePoints: ReadonlyArray<INodePoint>, startIndex: number, endIndex: number, state: LinkLabelCollectingState | null): {
+export declare function eatAndCollectLinkLabel(nodePoints: ReadonlyArray<INodePoint>, startIndex: number, endIndex: number, state: ILinkLabelCollectingState | null): {
     nextIndex: number;
-    state: LinkLabelCollectingState;
+    state: ILinkLabelCollectingState;
 };

package/lib/types/util/link-title.d.ts

@@ -5,7 +5,7 @@ import type { INodePoint } from '@yozora/character';
  *
  * @see https://github.github.com/gfm/#link-title
  */
-export interface LinkTitleCollectingState {
+export interface ILinkTitleCollectingState {
     /**
      * Whether the current token has collected a legal LinkDestination
      */
@@ -27,7 +27,7 @@ export interface LinkTitleCollectingState {
  * @param state
  * @see https://github.github.com/gfm/#link-title
  */
-export declare function eatAndCollectLinkTitle(nodePoints: ReadonlyArray<INodePoint>, startIndex: number, endIndex: number, state: LinkTitleCollectingState | null): {
+export declare function eatAndCollectLinkTitle(nodePoints: ReadonlyArray<INodePoint>, startIndex: number, endIndex: number, state: ILinkTitleCollectingState | null): {
     nextIndex: number;
-    state: LinkTitleCollectingState;
+    state: ILinkTitleCollectingState;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-definition",
-  "version": "2.0.0-alpha.0",
+  "version": "2.0.0",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -35,9 +35,9 @@
     "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.0.0-alpha.0",
-    "@yozora/character": "^2.0.0-alpha.0",
-    "@yozora/core-tokenizer": "^2.0.0-alpha.0"
+    "@yozora/ast": "^2.0.0",
+    "@yozora/character": "^2.0.0",
+    "@yozora/core-tokenizer": "^2.0.0"
   },
-  "gitHead": "0171501339c49ffd02ed16a63447fa20a47a29a7"
+  "gitHead": "65e99d1709fdd1c918465dce6b1e91de96bdab5e"
 }