amateras 0.5.0 → 0.6.0

package/ext/markdown/src/lib/type.ts

@@ -0,0 +1,26 @@
+export const INLINE = 'INLINE';
+export const BLOCK = 'BLOCK';
+export const TEXT = 'TEXT';
+export const IMAGE = 'IMAGE';
+export const LINK = 'LINK';
+export const QUICK_LINK = 'QUICK_LINK';
+export const CODE = 'CODE';
+export const ITALIC = 'ITALIC';
+export const BOLD = 'BOLD';
+export const TEXT_LINE = 'TEXT_LINE';
+export const HEADING = 'HEADING';
+export const CODE_START = 'CODE_START';
+export const CODE_LINE = 'CODE_LINE';
+export const CODE_END = 'CODE_END';
+export const UNORDERED_LIST_ITEM = 'UNORDERED_LIST_ITEM';
+export const ORDERED_LIST_ITEM = 'ORDERED_LIST_ITEM';
+export const BLOCKQUOTE = 'BLOCKQUOTE';
+export const ALERT = 'ALERT';
+export const ALERT_LINE = 'ALERT_LINE';
+export const HORIZONTAL_RULE = 'HORIZONTAL_RULE';
+export const TABLE = 'TABLE';
+export const TABLE_ROW = 'TABLE_ROW';
+export const TABLE_COLUMN = 'TABLE_COLUMN';
+export const EMPTY_LINE = 'EMPTY_LINE'
+export const INLINE_TEXT = "INLINE_TEXT";
+export const INLINE_CONTENT = "INLINE_CONTENT"

package/ext/markdown/src/lib/util.ts

@@ -0,0 +1,21 @@
+import type { BlockTokenizer, InlineTokenizer, MarkdownLexer } from "#structure/MarkdownLexer";
+import type { MarkdownParseProcessor, MarkdownParser } from "#structure/MarkdownParser";
+import { forEach } from "amateras/lib/native";
+
+export const setBlockTokenizer = (lexer: MarkdownLexer, type: string, tokenizer: BlockTokenizer) => lexer.blockTokenizers.set(type, tokenizer);
+export const setInlineTokenizer = (lexer: MarkdownLexer, type: string, tokenizer: InlineTokenizer) => lexer.inlineTokenizers.set(type, tokenizer);
+
+export const setProcessor = (parser: MarkdownParser, type: string, processor: MarkdownParseProcessor) => parser.processors.set(type, processor);
+
+export const htmltag = (tagname: string, content: string) => `<${tagname}>${content}</${tagname}>`
+
+export const htmlEscapeChar = (str: string) => {
+    forEach([
+        ['&', '&amp;'],
+        ['<', '&lt;'],
+        ['>', '&gt;'],
+        ['"', '&quot;'],
+        ["'", '&#39;']
+    ] as [string, string][], group => str = str.replaceAll(...group))
+    return str;
+}

package/ext/markdown/src/structure/Markdown.ts

@@ -0,0 +1,54 @@
+import { alertProcessor, alertTokenizer } from "../syntax/alert";
+import { blockquoteProcessor, blockquoteTokenizer } from "../syntax/blockquote";
+import { boldProcessor, boldTokenizer } from "../syntax/bold";
+import { codeProcessor, codeTokenizer } from "../syntax/code";
+import { codeblockProcessor, codeblockTokenizer } from "../syntax/codeblock";
+import { headingProcessor, headingTokenizer } from "../syntax/heading";
+import { horizontalRuleProcessor, horizontalRuleTokenizer } from "../syntax/horizontalRule";
+import { imageProcessor, imageTokenizer } from "../syntax/image";
+import { italicProcessor, italicTokenizer } from "../syntax/italic";
+import { linkProcessor, linkTokenizer } from "../syntax/link";
+import { listProcessor, listTokenizer } from "../syntax/list";
+import { textLineProcessor, textProcessor } from "../syntax/text";
+import { MarkdownLexer } from "./MarkdownLexer";
+import { MarkdownParser } from "./MarkdownParser";
+
+export class Markdown {
+    lexer = new MarkdownLexer();
+    parser = new MarkdownParser();
+    constructor() {
+        this.lexer.use(
+            headingTokenizer,
+            codeblockTokenizer,
+            listTokenizer,
+            alertTokenizer,
+            blockquoteTokenizer,
+            horizontalRuleTokenizer,
+            imageTokenizer, // image tokenizer must before link
+            linkTokenizer, // link tokenizer must before bold and italic and code
+            codeTokenizer,
+            boldTokenizer,
+            italicTokenizer,
+        )
+        
+        this.parser.use(
+            textProcessor,
+            imageProcessor,
+            linkProcessor,
+            codeProcessor,
+            italicProcessor,
+            boldProcessor,
+            textLineProcessor,
+            headingProcessor,
+            codeblockProcessor,
+            listProcessor,
+            alertProcessor,
+            blockquoteProcessor,
+            horizontalRuleProcessor
+        )
+    }
+
+    parseHTML(str: string) {
+        return this.parser.parse(this.lexer.blockTokenize(str));
+    }
+}

package/ext/markdown/src/structure/MarkdownLexer.ts

@@ -0,0 +1,111 @@
+import { BLOCK, EMPTY_LINE, INLINE_CONTENT, INLINE_TEXT, TEXT_LINE } from "#lib/type";
+import { forEach, isString } from "amateras/lib/native";
+
+export class MarkdownLexer {
+    blockTokenizers = new Map<string, BlockTokenizer>();
+    inlineTokenizers = new Map<string, InlineTokenizer>();
+    
+    blockTokenize(str: string) {
+        const lines = str.split(/\r?\n/);
+        const tokens: BlockToken[] = [];
+        let lineIndex = 0;
+        lineLoop: while (lineIndex < lines.length) {
+            let line = lines[lineIndex];
+            if (line === undefined) throw 'LINE ERROR';
+            let token: BlockToken | undefined;
+            for (const [type, tokenizer] of this.blockTokenizers) {
+                const matched = line.match(tokenizer.regex);
+                if (matched) {
+                    const {content, multiLine, data} = tokenizer.handle(matched, lineIndex, lines);
+                    token = { layout: BLOCK, type, content, data }
+                    if (multiLine) {
+                        tokens.push(token);
+                        tokens.push(...multiLine.tokens)
+                        lineIndex = multiLine.skip;
+                        continue lineLoop;
+                    }
+                    break;
+                }
+            }
+            if (!token) token = { 
+                layout: BLOCK, 
+                ...(
+                    line.length 
+                    ?   { type: TEXT_LINE, content: this.inlineTokenize(line) } 
+                    :   { type: EMPTY_LINE, content: [] }
+                )
+            };
+            tokens.push(token);
+            lineIndex++;
+        }
+        return tokens;
+    }
+
+    inlineTokenize(str: string): InlineToken[] {
+        const tokens: InlineToken[] = [];
+        let remainStr = str;
+        while (remainStr.length) {
+            let token: InlineToken | undefined;
+            for (const [type, tokenizer] of this.inlineTokenizers) {
+                const matched = remainStr.match(tokenizer.regex);
+                if (matched) {
+                    const {index, 0: matchStr} = matched;
+                    // handle before matched string
+                    if (index != 0) tokens.push(...this.inlineTokenize(remainStr.substring(0, index)));
+                    // handle matched string
+                    const {content, data} = tokenizer.handle(matched);
+                    token = { type, ...(isString(content) ? { layout: INLINE_TEXT, text: content } : { layout: INLINE_CONTENT, content })};
+                    if (data) token.data = data;
+                    remainStr = remainStr.substring(index! + matchStr.length);
+                    break;
+                }
+            }
+            if (!token) {
+                token = { type: 'TEXT', layout: INLINE_TEXT, text: remainStr };
+                remainStr = '';
+            }
+            tokens.push(token);
+        }
+        return tokens;
+    }
+
+    use(...handle: ((parser: this) => void)[]) {
+        forEach(handle, fn => fn(this));
+        return this;
+    }
+}
+
+export type BlockTokenizer = {
+    regex: RegExp;
+    handle: (matches: RegExpMatchArray, position: number, lines: string[]) => { content: (InlineToken | BlockToken)[], multiLine?: BlockTokenizerMultiLine, data?: {[key: string]: any} };
+}
+export type BlockTokenizerMultiLine = {
+    skip: number;
+    tokens: BlockToken[];
+}
+export type InlineTokenizer = {
+    regex: RegExp;
+    handle: (matches: RegExpMatchArray) => { content: InlineToken[] | string, data?: {[key: string]: any} }
+}
+
+export interface TokenBase {
+    type: string;
+    layout: 'BLOCK' | 'INLINE_CONTENT' | 'INLINE_TEXT';
+    content?: Token[];
+    text?: string;
+    data?: {[key: string]: any};
+}
+export interface BlockToken extends TokenBase {
+    layout: 'BLOCK'
+    content: Token[];
+}
+export interface InlineTextToken extends TokenBase {
+    layout: 'INLINE_TEXT'
+    text: string;
+}
+export interface InlineContentToken extends TokenBase {
+    layout: 'INLINE_CONTENT'
+    content: (InlineToken)[];
+}
+export type InlineToken = InlineTextToken | InlineContentToken;
+export type Token = BlockToken | InlineToken;

package/ext/markdown/src/structure/MarkdownParser.ts

@@ -0,0 +1,33 @@
+import { forEach, isString } from "../../../../src/lib/native";
+import { type Token } from "./MarkdownLexer";
+
+export class MarkdownParser {
+    processors = new Map<string, MarkdownParseProcessor>();
+    
+    parse(tokens: (Token)[]) {
+        let html = '';
+        let i = 0;
+        while (i < tokens.length) {
+            const token = tokens[i]!;
+            const processor = this.processors.get(token.type);
+            if (processor) {
+                const result = processor(token, tokens.slice(i));
+                if (isString(result)) {
+                    html += result;
+                } else {
+                    html += result.html;
+                    i += result.skipTokens;
+                }
+            }
+            i++;
+        }
+        return html;
+    }
+
+    use(...handle: ((parser: this) => void)[]) {
+        forEach(handle, fn => fn(this));
+        return this;
+    }
+}
+
+export type MarkdownParseProcessor = (token: Token, tokens: Token[]) => (string | { html: string, skipTokens: number })

package/ext/markdown/src/syntax/alert.ts

@@ -0,0 +1,46 @@
+import { ALERT, ALERT_LINE, BLOCK } from "#lib/type";
+import { setBlockTokenizer, setProcessor } from "#lib/util";
+import type { BlockToken, MarkdownLexer } from "#structure/MarkdownLexer";
+import type { MarkdownParser } from "#structure/MarkdownParser";
+import { uppercase } from "amateras/lib/uppercase";
+
+export const alertProcessor = (parser: MarkdownParser) => setProcessor(parser, ALERT, (token, tokens) => {
+    let html = '';
+    let i = 1;
+    while (i < tokens.length) {
+        const token = tokens[i]!;
+        if (token.type !== ALERT_LINE) break;
+        html += parser.parse(token.content![0]!.content!);
+        i++;
+    }
+    const alertType = token.data?.alertType as string;
+    return {
+        html: `<blockquote class="alert alert-${alertType}"><p class="alert-title">${uppercase(alertType, 0, 1)}</p>${html}</blockquote>`,
+        skipTokens: i
+    }
+})
+
+export const alertTokenizer = (lexer: MarkdownLexer) => setBlockTokenizer(lexer, ALERT, {
+    regex: /^> ?\[!(?:(?:NOTE)|(?:TIP)|(?:IMPORTANT)|(?:WARNING)|(?:CAUTION))\]/,
+    handle(_, position, lines) {
+        const tokens: BlockToken[] = [];
+        const match = lines[position]!.match(/> ?\[!(.+?)\]/);
+        const alertType = match?.[1]?.toLowerCase();
+        position++
+        while (position < lines.length) {
+            const line = lines[position]!;
+            const match = line.match(/^> ?(.+)/);
+            if (match) tokens.push({ layout: BLOCK, type: ALERT_LINE, content: lexer.blockTokenize(match[1]!) });
+            else break;
+            position++;
+        }
+        return {
+            content: [],
+            data: { alertType },
+            multiLine: {
+                skip: position,
+                tokens
+            }
+        }
+    },
+})

package/ext/markdown/src/syntax/blockquote.ts

@@ -0,0 +1,35 @@
+import { BLOCKQUOTE } from "#lib/type";
+import { htmltag, setBlockTokenizer, setProcessor } from "#lib/util";
+import type { MarkdownLexer } from "#structure/MarkdownLexer";
+import type { MarkdownParser } from "#structure/MarkdownParser";
+
+export const blockquoteProcessor = (parser: MarkdownParser) => setProcessor(parser, BLOCKQUOTE, (token, tokens) => {
+        let i = 0;
+        const blockquote = (deep: number) => {
+            let html = '';
+            while (i < tokens.length) {
+                const {type, content, data} = tokens[i]!;
+                if (type !== BLOCKQUOTE) break;
+                if (data!.deep > deep) html += blockquote(data!.deep);
+                else if (data!.deep < deep) break;
+                else { html += parser.parse(content!); i++ }
+            }
+            return htmltag('blockquote', html)
+        }
+        return {
+            html: blockquote(token.data!.deep),
+            skipTokens: i
+        }
+    })
+
+export const blockquoteTokenizer = (lexer: MarkdownLexer) => setBlockTokenizer(lexer, BLOCKQUOTE, {
+        regex: /^(>+) ?(.+)?/,
+        handle(matches) {
+            return {
+                content: lexer.blockTokenize(matches[2] ?? ''),
+                data: {
+                    deep: (matches[1]!.length - 1)
+                }
+            }
+        }
+    })

package/ext/markdown/src/syntax/bold.ts

@@ -0,0 +1,11 @@
+import { BOLD } from "#lib/type";
+import { htmltag, setInlineTokenizer, setProcessor } from "#lib/util";
+import type { MarkdownLexer } from "#structure/MarkdownLexer";
+import type { MarkdownParser } from "#structure/MarkdownParser";
+
+export const boldProcessor = (parser: MarkdownParser) => setProcessor(parser, BOLD, token => htmltag('b', parser.parse(token.content!)))
+
+export const boldTokenizer = (lexer: MarkdownLexer) => setInlineTokenizer(lexer, BOLD, {
+        regex: /\*\*(.+?\*?)\*\*/,
+        handle: matches => ({ content: lexer.inlineTokenize(matches[1]!) })
+    })

package/ext/markdown/src/syntax/code.ts

@@ -0,0 +1,11 @@
+import { CODE } from "#lib/type";
+import { htmlEscapeChar, htmltag, setInlineTokenizer, setProcessor } from "#lib/util";
+import type { MarkdownLexer } from "#structure/MarkdownLexer";
+import type { MarkdownParser } from "#structure/MarkdownParser";
+
+export const codeProcessor = (parser: MarkdownParser) => setProcessor(parser, CODE, token => htmltag('code', htmlEscapeChar(token.text!)))
+
+export const codeTokenizer = (lexer: MarkdownLexer) => setInlineTokenizer(lexer, CODE, {
+    regex: /`(.+?)`/,
+    handle: matches => ({ content: matches[1]! })
+})

package/ext/markdown/src/syntax/codeblock.ts

@@ -0,0 +1,44 @@
+import { BLOCK, CODE_END, CODE_LINE, CODE_START } from "#lib/type";
+import { htmlEscapeChar, setBlockTokenizer, setProcessor } from "#lib/util";
+import type { BlockToken, MarkdownLexer } from "#structure/MarkdownLexer";
+import type { MarkdownParser } from "#structure/MarkdownParser";
+
+export const codeblockProcessor = (parser: MarkdownParser) => setProcessor(parser, CODE_START, (token, tokens) => {
+        let html = '';
+        let i = 1;
+        while (i < tokens.length) {
+            const token = tokens[i]!;
+            if (token.type === CODE_END) break;
+            html += token.content![0]!.text;
+            i++;
+        }
+        return {
+            html: `<pre><code${token.data?.lang ? ` lang="${token.data.lang}"` : ''}>${htmlEscapeChar(html)}</code></pre>`,
+            skipTokens: i
+        }
+    })
+
+export const codeblockTokenizer = (lexer: MarkdownLexer) => setBlockTokenizer(lexer, CODE_START, {
+    regex: /^```(\w+)/,
+    handle: (matches, position, lines) => {
+        const tokens: BlockToken[] = [];
+        position++;
+        while (position < lines.length) {
+            const line = lines[position]!;
+            position++;
+            if (line.includes('```')) {
+                tokens.push({ layout: BLOCK, type: CODE_END, content: [] })
+                break;
+            }
+            tokens.push({ layout: BLOCK, type: CODE_LINE, content: [{ layout: "INLINE_TEXT", type: 'CODE_TEXT', text: `${line}\n` }] });
+        }
+        return {
+            content: [],
+            data: { lang: matches[1] },
+            multiLine: {
+                skip: position,
+                tokens
+            }
+        }
+    }
+})

package/ext/markdown/src/syntax/heading.ts

@@ -0,0 +1,14 @@
+import { HEADING } from "#lib/type";
+import { htmltag, setBlockTokenizer, setProcessor } from "#lib/util";
+import type { MarkdownLexer } from "#structure/MarkdownLexer";
+import type { MarkdownParser } from "#structure/MarkdownParser";
+
+export const headingProcessor = (parser: MarkdownParser) => setProcessor(parser, HEADING, token => {
+    const tagname = `h${token.data!.level}`;
+    return htmltag(tagname, parser.parse(token.content!))
+})
+
+export const headingTokenizer = (lexer: MarkdownLexer) => setBlockTokenizer(lexer, HEADING, {
+    regex: /^(#+) (.+)/,
+    handle: matches => ({ content: lexer.inlineTokenize(matches[2]!), data: { level: matches[1]!.length } })
+})

package/ext/markdown/src/syntax/horizontalRule.ts

@@ -0,0 +1,11 @@
+import { HORIZONTAL_RULE } from "#lib/type";
+import { setBlockTokenizer, setProcessor } from "#lib/util";
+import type { MarkdownLexer } from "#structure/MarkdownLexer";
+import type { MarkdownParser } from "#structure/MarkdownParser";
+
+export const horizontalRuleProcessor = (parser: MarkdownParser) => setProcessor(parser, HORIZONTAL_RULE, _ => `<hr>`)
+
+export const horizontalRuleTokenizer = (lexer: MarkdownLexer) => setBlockTokenizer(lexer, HORIZONTAL_RULE, {
+    regex: /^---/,
+    handle: _ => ({ content: [] })
+})

package/ext/markdown/src/syntax/image.ts

@@ -0,0 +1,23 @@
+import { IMAGE } from "#lib/type";
+import { setInlineTokenizer, setProcessor } from "#lib/util";
+import type { MarkdownLexer } from "#structure/MarkdownLexer";
+import type { MarkdownParser } from "#structure/MarkdownParser";
+
+export const imageProcessor = (parser: MarkdownParser) => setProcessor(parser, IMAGE, token => {
+    const { url, title } = token.data!;
+    return `<img alt="${parser.parse(token.content!)}" src="${url}"${title ? ` title="${title}"` : ''}>`
+})
+
+export const imageTokenizer = (lexer: MarkdownLexer) => setInlineTokenizer(lexer, IMAGE, {
+    regex: /^!\[(.+?)\]\((.+?)\)/,
+    handle: matches => {
+        const [_, alt, detail] = matches as [string, string, string];
+        const [__, url, title] = detail.match(/(\w\w+?:\/\/[^\s]+)(?: "(.+?)")?/) as [string, string, string];
+        return {
+            content: lexer.inlineTokenize(alt),
+            data: {
+                url, title
+            }
+        }
+    }
+})

package/ext/markdown/src/syntax/italic.ts

@@ -0,0 +1,11 @@
+import { ITALIC } from "#lib/type";
+import { setInlineTokenizer, setProcessor } from "#lib/util";
+import type { MarkdownLexer } from "#structure/MarkdownLexer";
+import type { MarkdownParser } from "#structure/MarkdownParser";
+
+export const italicProcessor = (parser: MarkdownParser) => setProcessor(parser, ITALIC, token => `<i>${parser.parse(token.content!)}</i>`)
+
+export const italicTokenizer = (lexer: MarkdownLexer) => setInlineTokenizer(lexer, ITALIC, {
+    regex: /\*(.+?)\*/,
+    handle: matches => ({ content: lexer.inlineTokenize(matches[1]!) })
+})

package/ext/markdown/src/syntax/link.ts

@@ -0,0 +1,46 @@
+import { LINK, QUICK_LINK } from "#lib/type";
+import { setInlineTokenizer, setProcessor } from "#lib/util";
+import type { MarkdownLexer, Token } from "#structure/MarkdownLexer";
+import type { MarkdownParser } from "#structure/MarkdownParser";
+import { isUndefined } from "amateras/lib/native";
+
+export const linkProcessor = (parser: MarkdownParser) => {
+    const linkProcessor = (token: Token) => {
+        const {href, email, title} = token.data!;
+        return `<a href="${isUndefined(href) ? `mailto:${email}` : href}"${title ? ` title="${title}"` : ''}>${parser.parse(token.content!)}</a>`
+    }
+    setProcessor(parser, QUICK_LINK, linkProcessor)
+    setProcessor(parser, LINK, linkProcessor)
+}
+
+export const linkTokenizer = (lexer: MarkdownLexer) => {
+
+    setInlineTokenizer(lexer, LINK, {
+        regex: /\[(.+?)\]\(((?:\w+?@(?:\w|\.\w)+)|(?:\w\w+?:[^\s)]+))(?: "(.+)?")?\)/,
+        handle: matches => {
+            const [_, alt, detail, title] = matches as [string, string, string, string];
+            const match = detail.match(/(?:\w+?@(?:\w|\.\w)+)|(?:\w\w+?:\/\/[^\s]+)/);
+            const [resolver] = match!;
+            const email_or_href = resolver.includes('@') ? { email: resolver }: { href: resolver };
+            return {
+                content: lexer.inlineTokenize(alt),
+                data: {
+                    title, ...email_or_href
+                }
+            }
+        }
+    })
+    setInlineTokenizer(lexer, QUICK_LINK, {
+        regex: /<((?:\w+?@(?:\w|\.\w)+)|(?:\w\w+?:[^\s>]+))>/,
+        handle: matches => {
+            const [_, detail] = matches as [string, string];
+            const match = detail.match(/(?:\w+?@(?:\w|\.\w)+)|(?:\w\w+?:\/\/[^\s]+)/);
+            const [resolver] = match!;
+            const email_or_href = resolver.includes('@') ? { email: resolver }: { href: resolver };
+            return {
+                content: resolver,
+                data: email_or_href
+            }
+        }
+    })
+}

package/ext/markdown/src/syntax/list.ts

@@ -0,0 +1,121 @@
+import { EMPTY_LINE, ORDERED_LIST_ITEM, TEXT_LINE, UNORDERED_LIST_ITEM } from "#lib/type";
+import { htmltag, setBlockTokenizer, setProcessor } from "#lib/util";
+import type { MarkdownLexer, Token } from "#structure/MarkdownLexer";
+import type { MarkdownParser } from "#structure/MarkdownParser";
+import { equal, isString } from "amateras/lib/native";
+
+export const listProcessor = (parser: MarkdownParser) => {
+    const listType = (type: string) => type === ORDERED_LIST_ITEM ? 'ol' : 'ul'
+    const listProcessor = (token: Token, tokens: Token[]) => {
+        let i = 0;
+        // cache the list by deep number
+        const deepListMap = new Map<number, List>();
+
+        const listGenerator = (type: string, deep: number) => {
+            const getList = deepListMap.get(deep)
+            const list = getList && listType(type) === getList.tagname ? getList : List(listType(type), []);
+            deepListMap.set(deep, list);
+
+            while (i < tokens.length) {
+                const token = tokens[i]!;
+                const tokenType = token.type;
+                // if token type not equal list item / empty line / text line, then finish loop
+                if (!equal(tokenType, ORDERED_LIST_ITEM, UNORDERED_LIST_ITEM, EMPTY_LINE, TEXT_LINE)) { i--; break};
+                // if token type equal text line
+                if (tokenType === TEXT_LINE) {
+                    const text = token.content![0]?.text;
+                    // if text start with double space
+                    if (text?.match(/^\s\s/)) {
+                        const match = text.match(/^(\s+)(.+)?/)!;
+                        // if no content, then next token
+                        if (!match[2]) { i++; continue }
+                        token.data = { deep: Math.trunc(match[1]!.length / 2) - 1 }
+                    }
+                    // if text start with tab
+                    else if (text?.match(/^\t/)) {
+                        const match = text.match(/^(\t+)(.+)?/)!;
+                        // if no content, then next token
+                        if (!match[2]) { i++; continue }
+                        token.data = { deep: match[1]!.length - 1 }
+                    }
+                    // else break 
+                    else {i--; break};
+                }
+                // if token type equal empty line, jump to next token
+                if (tokenType === EMPTY_LINE) i++;
+                // if token deep number not equal latest deep of list
+                else if (token.data!.deep !== deep) {
+                    // if bigger, push deeper list into current list item
+                    if (token.data!.deep > deep) deepListMap.get(deep)?.items.at(-1)?.content.push(listGenerator(tokenType, token.data!.deep))
+                    // else delete current deep cache and return to upper list
+                    else { deepListMap.delete(deep); break; }
+                }
+                // if token type equal text line, convert this list to paragraph mode
+                else if (tokenType === TEXT_LINE) {
+                    list.paragraph = true;
+                    list.items.at(-1)?.content.push(parser.parse(token.content!))
+                    i++
+                }
+                // if list type not equal, then finish loop
+                else if (tokenType !== type) {
+                    deepListMap.delete(deep);
+                    break;
+                }
+                // push list item
+                else {
+                    list.items.push(ListItem([parser.parse(token.content!)]));
+                    i++
+                }
+            }
+            return list;
+        }
+
+        return {
+            html: `${listGenerator(token.type, token.data!.deep)}`,
+            skipTokens: i
+        }
+    }
+
+    interface ListItem { content: (string | List)[], toString(): string }
+    const ListItem = (content: (string | List)[]): ListItem => ({
+        content: content,
+        toString() { return htmltag('li', this.content.join('')) }
+    })
+
+    interface List { tagname: string, items: ListItem[], paragraph: boolean, toString(): string }
+    const List = (tagname: 'ul' | 'ol', items: ListItem[]): List => ({
+        tagname: tagname,
+        items: items,
+        paragraph: false,
+        toString() {
+            if (this.paragraph) this.items.forEach(item => item.content.forEach((text, i) => isString(text) && (item.content[i] = htmltag('p', text))))
+            return htmltag(this.tagname, this.items.join(''))
+        }
+    })
+
+    setProcessor(parser, UNORDERED_LIST_ITEM, listProcessor);
+    setProcessor(parser, ORDERED_LIST_ITEM, listProcessor);
+}
+
+export const listTokenizer = (lexer: MarkdownLexer) => {
+    const listHandle = (matches: RegExpMatchArray) => {
+        const prefix = matches[0].split(/[-*]/)[0]!;
+        const spaces = prefix.match(/\s/)?.length ?? 0;
+        const tabs = prefix.match(/\t/)?.length ?? 0;
+        return ({
+        content: lexer.inlineTokenize(matches[1]!),
+        data: {
+            deep: Math.trunc(tabs + spaces / 2)
+        }
+    })
+    }
+    setBlockTokenizer(lexer, UNORDERED_LIST_ITEM, {
+        regex: /^(?:[\s\t]+)?[-*] (.+)/,
+        handle: listHandle
+    })
+
+    setBlockTokenizer(lexer, ORDERED_LIST_ITEM, {
+        regex: /^(?:[\s\t]+)?\d+\. (.+)/,
+        handle: listHandle
+    })
+}