@yozora/tokenizer-autolink 2.0.0-alpha.0 → 2.0.0

package/README.md

@@ -84,14 +84,14 @@ so you can use `YozoraParser` / `GfmExParser` / `GfmParser` directly.
 registered in *YastParser* as a plugin-in before it can be used.
 
 ```typescript {4,9}
-import { DefaultYastParser } from '@yozora/core-parser'
+import { DefaultParser } from '@yozora/core-parser'
 import ParagraphTokenizer from '@yozora/tokenizer-paragraph'
 import TextTokenizer from '@yozora/tokenizer-text'
 import AutolinkTokenizer from '@yozora/tokenizer-autolink'
 
-const parser = new DefaultYastParser()
-  .useBlockFallbackTokenizer(new ParagraphTokenizer())
-  .useInlineFallbackTokenizer(new TextTokenizer())
+const parser = new DefaultParser()
+  .useFallbackTokenizer(new ParagraphTokenizer())
+  .useFallbackTokenizer(new TextTokenizer())
   .useTokenizer(new AutolinkTokenizer())
 
 // parse source markdown content
@@ -241,7 +241,6 @@ Name              | Type        | Required  | Default
 [@yozora/tokenizer-link]:                     https://github.com/yozorajs/yozora/tree/main/tokenizers/link#readme
 [@yozora/tokenizer-link-reference]:           https://github.com/yozorajs/yozora/tree/main/tokenizers/link-reference#readme
 [@yozora/tokenizer-list]:                     https://github.com/yozorajs/yozora/tree/main/tokenizers/list#readme
-[@yozora/tokenizer-list-item]:                https://github.com/yozorajs/yozora/tree/main/tokenizers/list-item#readme
 [@yozora/tokenizer-math]:                     https://github.com/yozorajs/yozora/tree/main/tokenizers/math#readme
 [@yozora/tokenizer-paragraph]:                https://github.com/yozorajs/yozora/tree/main/tokenizers/paragraph#readme
 [@yozora/tokenizer-setext-heading]:           https://github.com/yozorajs/yozora/tree/main/tokenizers/setext-heading#readme
@@ -301,7 +300,6 @@ Name              | Type        | Required  | Default
 [doc-@yozora/tokenizer-definition]:           https://yozora.guanghechen.com/docs/package/tokenizer-definition
 [doc-@yozora/tokenizer-link-reference]:       https://yozora.guanghechen.com/docs/package/tokenizer-link-reference
 [doc-@yozora/tokenizer-list]:                 https://yozora.guanghechen.com/docs/package/tokenizer-list
-[doc-@yozora/tokenizer-list-item]:            https://yozora.guanghechen.com/docs/package/tokenizer-list-item
 [doc-@yozora/tokenizer-math]:                 https://yozora.guanghechen.com/docs/package/tokenizer-math
 [doc-@yozora/tokenizer-paragraph]:            https://yozora.guanghechen.com/docs/package/tokenizer-paragraph
 [doc-@yozora/tokenizer-setext-heading]:       https://yozora.guanghechen.com/docs/package/tokenizer-setext-heading

package/lib/cjs/index.js

@@ -103,12 +103,79 @@ function eatAutolinkSchema(nodePoints, startIndex, endIndex) {
     return { valid: true, nextIndex: i };
 }
 
-const uniqueName = '@yozora/tokenizer-autolink';
-
 const helpers = [
     { contentType: 'uri', eat: eatAbsoluteUri },
     { contentType: 'email', eat: eatEmailAddress },
 ];
+const match = function (api) {
+    return {
+        findDelimiter: () => coreTokenizer.genFindDelimiter(_findDelimiter),
+        processSingleDelimiter,
+    };
+    function _findDelimiter(startIndex, endIndex) {
+        const nodePoints = api.getNodePoints();
+        for (let i = startIndex; i < endIndex; ++i) {
+            if (nodePoints[i].codePoint !== character.AsciiCodePoint.OPEN_ANGLE)
+                continue;
+            let nextIndex = endIndex;
+            let contentType = null;
+            for (const helper of helpers) {
+                const eatResult = helper.eat(nodePoints, i + 1, endIndex);
+                nextIndex = Math.min(nextIndex, eatResult.nextIndex);
+                if (eatResult.valid) {
+                    contentType = helper.contentType;
+                    nextIndex = eatResult.nextIndex;
+                    break;
+                }
+            }
+            if (contentType == null) {
+                i = Math.max(i, nextIndex - 1);
+                continue;
+            }
+            if (nextIndex < endIndex && nodePoints[nextIndex].codePoint === character.AsciiCodePoint.CLOSE_ANGLE) {
+                return {
+                    type: 'full',
+                    startIndex: i,
+                    endIndex: nextIndex + 1,
+                    contentType,
+                };
+            }
+            i = nextIndex - 1;
+        }
+        return null;
+    }
+    function processSingleDelimiter(delimiter) {
+        const token = {
+            nodeType: ast.LinkType,
+            startIndex: delimiter.startIndex,
+            endIndex: delimiter.endIndex,
+            contentType: delimiter.contentType,
+            children: api.resolveFallbackTokens([], delimiter.startIndex + 1, delimiter.endIndex - 1),
+        };
+        return [token];
+    }
+};
+
+const parse = function (api) {
+    return {
+        parse: tokens => tokens.map(token => {
+            const nodePoints = api.getNodePoints();
+            let url = character.calcStringFromNodePoints(nodePoints, token.startIndex + 1, token.endIndex - 1);
+            if (token.contentType === 'email') {
+                url = 'mailto:' + url;
+            }
+            const encodedUrl = coreTokenizer.encodeLinkDestination(url);
+            const children = api.parseInlineTokens(token.children);
+            const node = api.shouldReservePosition
+                ? { type: ast.LinkType, position: api.calcPosition(token), url: encodedUrl, children }
+                : { type: ast.LinkType, url: encodedUrl, children };
+            return node;
+        }),
+    };
+};
+
+const uniqueName = '@yozora/tokenizer-autolink';
+
 class AutolinkTokenizer extends coreTokenizer.BaseInlineTokenizer {
     constructor(props = {}) {
         var _a, _b;
@@ -116,76 +183,15 @@ class AutolinkTokenizer extends coreTokenizer.BaseInlineTokenizer {
             name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
             priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.ATOMIC,
         });
-        this.match = api => {
-            return {
-                findDelimiter: () => coreTokenizer.genFindDelimiter(_findDelimiter),
-                processSingleDelimiter,
-            };
-            function _findDelimiter(startIndex, endIndex) {
-                const nodePoints = api.getNodePoints();
-                for (let i = startIndex; i < endIndex; ++i) {
-                    if (nodePoints[i].codePoint !== character.AsciiCodePoint.OPEN_ANGLE)
-                        continue;
-                    let nextIndex = endIndex;
-                    let contentType = null;
-                    for (const helper of helpers) {
-                        const eatResult = helper.eat(nodePoints, i + 1, endIndex);
-                        nextIndex = Math.min(nextIndex, eatResult.nextIndex);
-                        if (eatResult.valid) {
-                            contentType = helper.contentType;
-                            nextIndex = eatResult.nextIndex;
-                            break;
-                        }
-                    }
-                    if (contentType == null) {
-                        i = Math.max(i, nextIndex - 1);
-                        continue;
-                    }
-                    if (nextIndex < endIndex &&
-                        nodePoints[nextIndex].codePoint === character.AsciiCodePoint.CLOSE_ANGLE) {
-                        return {
-                            type: 'full',
-                            startIndex: i,
-                            endIndex: nextIndex + 1,
-                            contentType,
-                        };
-                    }
-                    i = nextIndex - 1;
-                }
-                return null;
-            }
-            function processSingleDelimiter(delimiter) {
-                const token = {
-                    nodeType: ast.LinkType,
-                    startIndex: delimiter.startIndex,
-                    endIndex: delimiter.endIndex,
-                    contentType: delimiter.contentType,
-                    children: api.resolveFallbackTokens([], delimiter.startIndex + 1, delimiter.endIndex - 1),
-                };
-                return [token];
-            }
-        };
-        this.parse = api => ({
-            parse: (token, children) => {
-                const nodePoints = api.getNodePoints();
-                let url = character.calcStringFromNodePoints(nodePoints, token.startIndex + 1, token.endIndex - 1);
-                if (token.contentType === 'email') {
-                    url = 'mailto:' + url;
-                }
-                const encodedUrl = coreTokenizer.encodeLinkDestination(url);
-                const result = {
-                    type: ast.LinkType,
-                    url: encodedUrl,
-                    children,
-                };
-                return result;
-            },
-        });
+        this.match = match;
+        this.parse = parse;
     }
 }
 
 exports.AutolinkTokenizer = AutolinkTokenizer;
 exports.AutolinkTokenizerName = uniqueName;
+exports.autolinkMatch = match;
+exports.autolinkParse = parse;
 exports["default"] = AutolinkTokenizer;
 exports.eatAbsoluteUri = eatAbsoluteUri;
 exports.eatAutolinkSchema = eatAutolinkSchema;

package/lib/esm/index.js

@@ -1,6 +1,6 @@
 import { isAsciiLetter, isAsciiDigitCharacter, AsciiCodePoint, isAlphanumeric, isAsciiCharacter, isWhitespaceCharacter, isAsciiControlCharacter, calcStringFromNodePoints } from '@yozora/character';
 import { LinkType } from '@yozora/ast';
-import { BaseInlineTokenizer, TokenizerPriority, genFindDelimiter, encodeLinkDestination } from '@yozora/core-tokenizer';
+import { genFindDelimiter, encodeLinkDestination, BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
 
 function eatEmailAddress(nodePoints, startIndex, endIndex) {
     let i = startIndex;
@@ -99,12 +99,79 @@ function eatAutolinkSchema(nodePoints, startIndex, endIndex) {
     return { valid: true, nextIndex: i };
 }
 
-const uniqueName = '@yozora/tokenizer-autolink';
-
 const helpers = [
     { contentType: 'uri', eat: eatAbsoluteUri },
     { contentType: 'email', eat: eatEmailAddress },
 ];
+const match = function (api) {
+    return {
+        findDelimiter: () => genFindDelimiter(_findDelimiter),
+        processSingleDelimiter,
+    };
+    function _findDelimiter(startIndex, endIndex) {
+        const nodePoints = api.getNodePoints();
+        for (let i = startIndex; i < endIndex; ++i) {
+            if (nodePoints[i].codePoint !== AsciiCodePoint.OPEN_ANGLE)
+                continue;
+            let nextIndex = endIndex;
+            let contentType = null;
+            for (const helper of helpers) {
+                const eatResult = helper.eat(nodePoints, i + 1, endIndex);
+                nextIndex = Math.min(nextIndex, eatResult.nextIndex);
+                if (eatResult.valid) {
+                    contentType = helper.contentType;
+                    nextIndex = eatResult.nextIndex;
+                    break;
+                }
+            }
+            if (contentType == null) {
+                i = Math.max(i, nextIndex - 1);
+                continue;
+            }
+            if (nextIndex < endIndex && nodePoints[nextIndex].codePoint === AsciiCodePoint.CLOSE_ANGLE) {
+                return {
+                    type: 'full',
+                    startIndex: i,
+                    endIndex: nextIndex + 1,
+                    contentType,
+                };
+            }
+            i = nextIndex - 1;
+        }
+        return null;
+    }
+    function processSingleDelimiter(delimiter) {
+        const token = {
+            nodeType: LinkType,
+            startIndex: delimiter.startIndex,
+            endIndex: delimiter.endIndex,
+            contentType: delimiter.contentType,
+            children: api.resolveFallbackTokens([], delimiter.startIndex + 1, delimiter.endIndex - 1),
+        };
+        return [token];
+    }
+};
+
+const parse = function (api) {
+    return {
+        parse: tokens => tokens.map(token => {
+            const nodePoints = api.getNodePoints();
+            let url = calcStringFromNodePoints(nodePoints, token.startIndex + 1, token.endIndex - 1);
+            if (token.contentType === 'email') {
+                url = 'mailto:' + url;
+            }
+            const encodedUrl = encodeLinkDestination(url);
+            const children = api.parseInlineTokens(token.children);
+            const node = api.shouldReservePosition
+                ? { type: LinkType, position: api.calcPosition(token), url: encodedUrl, children }
+                : { type: LinkType, url: encodedUrl, children };
+            return node;
+        }),
+    };
+};
+
+const uniqueName = '@yozora/tokenizer-autolink';
+
 class AutolinkTokenizer extends BaseInlineTokenizer {
     constructor(props = {}) {
         var _a, _b;
@@ -112,72 +179,9 @@ class AutolinkTokenizer extends BaseInlineTokenizer {
             name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
             priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
         });
-        this.match = api => {
-            return {
-                findDelimiter: () => genFindDelimiter(_findDelimiter),
-                processSingleDelimiter,
-            };
-            function _findDelimiter(startIndex, endIndex) {
-                const nodePoints = api.getNodePoints();
-                for (let i = startIndex; i < endIndex; ++i) {
-                    if (nodePoints[i].codePoint !== AsciiCodePoint.OPEN_ANGLE)
-                        continue;
-                    let nextIndex = endIndex;
-                    let contentType = null;
-                    for (const helper of helpers) {
-                        const eatResult = helper.eat(nodePoints, i + 1, endIndex);
-                        nextIndex = Math.min(nextIndex, eatResult.nextIndex);
-                        if (eatResult.valid) {
-                            contentType = helper.contentType;
-                            nextIndex = eatResult.nextIndex;
-                            break;
-                        }
-                    }
-                    if (contentType == null) {
-                        i = Math.max(i, nextIndex - 1);
-                        continue;
-                    }
-                    if (nextIndex < endIndex &&
-                        nodePoints[nextIndex].codePoint === AsciiCodePoint.CLOSE_ANGLE) {
-                        return {
-                            type: 'full',
-                            startIndex: i,
-                            endIndex: nextIndex + 1,
-                            contentType,
-                        };
-                    }
-                    i = nextIndex - 1;
-                }
-                return null;
-            }
-            function processSingleDelimiter(delimiter) {
-                const token = {
-                    nodeType: LinkType,
-                    startIndex: delimiter.startIndex,
-                    endIndex: delimiter.endIndex,
-                    contentType: delimiter.contentType,
-                    children: api.resolveFallbackTokens([], delimiter.startIndex + 1, delimiter.endIndex - 1),
-                };
-                return [token];
-            }
-        };
-        this.parse = api => ({
-            parse: (token, children) => {
-                const nodePoints = api.getNodePoints();
-                let url = calcStringFromNodePoints(nodePoints, token.startIndex + 1, token.endIndex - 1);
-                if (token.contentType === 'email') {
-                    url = 'mailto:' + url;
-                }
-                const encodedUrl = encodeLinkDestination(url);
-                const result = {
-                    type: LinkType,
-                    url: encodedUrl,
-                    children,
-                };
-                return result;
-            },
-        });
+        this.match = match;
+        this.parse = parse;
     }
 }
 
-export { AutolinkTokenizer, uniqueName as AutolinkTokenizerName, AutolinkTokenizer as default, eatAbsoluteUri, eatAutolinkSchema, eatEmailAddress };
+export { AutolinkTokenizer, uniqueName as AutolinkTokenizerName, match as autolinkMatch, parse as autolinkParse, AutolinkTokenizer as default, eatAbsoluteUri, eatAutolinkSchema, eatEmailAddress };

package/lib/types/index.d.ts

@@ -1,5 +1,7 @@
 export * from './util/email';
 export * from './util/uri';
+export { match as autolinkMatch } from './match';
+export { parse as autolinkParse } from './parse';
 export { AutolinkTokenizer, AutolinkTokenizer as default } from './tokenizer';
 export { uniqueName as AutolinkTokenizerName } from './types';
-export type { IToken as IAutolinkToken, ITokenizerProps as IAutolinkTokenizerProps, AutolinkContentType, } from './types';
+export type { IThis as IAutolinkHookContext, IToken as IAutolinkToken, ITokenizerProps as IAutolinkTokenizerProps, AutolinkContentType, } from './types';

package/lib/types/match.d.ts

@@ -0,0 +1,9 @@
+import type { IMatchInlineHookCreator } from '@yozora/core-tokenizer';
+import type { IDelimiter, IThis, IToken, T } from './types';
+/**
+ * Autolinks are absolute URIs and email addresses inside '<' and '>'.
+ * They are parsed as links, with the URL or email address as the link label.
+ *
+ * @see https://github.github.com/gfm/#autolink
+ */
+export declare const match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis>;

package/lib/types/parse.d.ts

@@ -0,0 +1,3 @@
+import type { IParseInlineHookCreator } from '@yozora/core-tokenizer';
+import type { INode, IThis, IToken, T } from './types';
+export declare const parse: IParseInlineHookCreator<T, IToken, INode, IThis>;

package/lib/types/tokenizer.d.ts

@@ -1,16 +1,12 @@
 import type { IInlineTokenizer, IMatchInlineHookCreator, IParseInlineHookCreator } from '@yozora/core-tokenizer';
 import { BaseInlineTokenizer } from '@yozora/core-tokenizer';
-import type { IDelimiter, INode, IToken, ITokenizerProps, T } from './types';
+import type { IDelimiter, INode, IThis, IToken, ITokenizerProps, T } from './types';
 /**
  * Lexical Analyzer for Autolink.
- *
- * Autolinks are absolute URIs and email addresses inside '<' and '>'.
- * They are parsed as links, with the URL or email address as the link label.
- *
  * @see https://github.github.com/gfm/#autolink
  */
-export declare class AutolinkTokenizer extends BaseInlineTokenizer<T, IDelimiter, IToken, INode> implements IInlineTokenizer<T, IDelimiter, IToken, INode> {
+export declare class AutolinkTokenizer extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis> implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis> {
     constructor(props?: ITokenizerProps);
-    readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken>;
-    readonly parse: IParseInlineHookCreator<T, IToken, INode>;
+    readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis>;
+    readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis>;
 }

package/lib/types/types.d.ts

@@ -1,9 +1,9 @@
-import type { ILink, LinkType } from '@yozora/ast';
+import type { Link, LinkType } from '@yozora/ast';
 import type { INodePoint } from '@yozora/character';
-import type { IBaseInlineTokenizerProps, IPartialYastInlineToken, IResultOfRequiredEater, IYastTokenDelimiter } from '@yozora/core-tokenizer';
+import type { IBaseInlineTokenizerProps, IPartialYastInlineToken, IResultOfRequiredEater, ITokenizer, IYastTokenDelimiter } from '@yozora/core-tokenizer';
 export declare type AutolinkContentType = 'uri' | 'email';
 export declare type T = LinkType;
-export declare type INode = ILink;
+export declare type INode = Link;
 export declare const uniqueName = "@yozora/tokenizer-autolink";
 export interface IToken extends IPartialYastInlineToken<T> {
     /**
@@ -18,6 +18,7 @@ export interface IDelimiter extends IYastTokenDelimiter {
      */
     contentType: AutolinkContentType;
 }
+export declare type IThis = ITokenizer;
 export declare type ITokenizerProps = Partial<IBaseInlineTokenizerProps>;
 export declare type ContentEater = (nodePoints: ReadonlyArray<INodePoint>, startIndex: number, endIndex: number) => IResultOfRequiredEater;
 export interface IContentHelper {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-autolink",
-  "version": "2.0.0-alpha.0",
+  "version": "2.0.0",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -35,9 +35,9 @@
     "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.0.0-alpha.0",
-    "@yozora/character": "^2.0.0-alpha.0",
-    "@yozora/core-tokenizer": "^2.0.0-alpha.0"
+    "@yozora/ast": "^2.0.0",
+    "@yozora/character": "^2.0.0",
+    "@yozora/core-tokenizer": "^2.0.0"
   },
-  "gitHead": "0171501339c49ffd02ed16a63447fa20a47a29a7"
+  "gitHead": "65e99d1709fdd1c918465dce6b1e91de96bdab5e"
 }