amateras 0.4.2 → 0.6.0

package/ext/idb/src/structure/builder/$IDBBuilder.ts

@@ -0,0 +1,230 @@
+import { $IDB, type $IDBConfig } from "#structure/$IDB";
+import { _Array_from, _instanceof, _JSON_stringify, _null, _Object_assign, _Object_fromEntries, _Promise, forEach, isFunction } from "amateras/lib/native";
+import { $IDBStoreBuilder } from "./$IDBStoreBuilder";
+import type { $IDBIndexConfig } from "#structure/$IDBIndex";
+import type { $IDBStoreConfig } from "#structure/$IDBStore";
+import { trycatch } from "amateras/lib/trycatch";
+// optimizer variables
+const objectStoreNames = 'objectStoreNames';
+const deleteObjectStore = 'deleteObjectStore';
+const createObjectStore = 'createObjectStore';
+const _indexedDB = indexedDB;
+const onupgradeneeded = 'onupgradeneeded';
+const onsuccess = 'onsuccess';
+
+export interface $IDBBuilder {
+    readonly name: string;
+    readonly version: number;
+}
+export class $IDBBuilder<Config extends $IDBConfig = { name: string, stores: {}, version: number }> {
+    #deleteUnused = false;
+    storeMap = new Map<string, $IDBStoreBuilder>();
+    #devMode: boolean = false;
+    constructor(config: Config) {
+        _Object_assign(this, config);
+    }
+
+    /**
+     * This option helping developer to debug when initializing.
+     * @param dev - Enable dev mode
+     */
+    devMode(dev: boolean) {
+        this.#devMode = dev;
+        return this;
+    }
+    
+    /**
+     * If set to true, unused store will be deleted when initialize.
+     * @param enable - Enable delete unused stores
+     */
+    deleteUnused(enable: boolean) {
+        this.#deleteUnused = enable;
+        return this;
+    }
+
+    /**
+     * Add new store to builder.
+     * @param name - Store name
+     * @param builder - Store builder or builder function
+     */
+    store<N extends string, B extends $IDBStoreBuilderFunction>(name: N, builder: B): $IDBBuilder<Prettify<Config & { stores: Config['stores'] & Prettify<Record<N, ReturnType<B>['config']>> }>>
+    store<N extends string, B extends $IDBStoreBuilder<any>>(name: N, builder: B): $IDBBuilder<Prettify<Config & { stores: Config['stores'] & Prettify<Record<N, B['config']>> }>>
+    store(name: string, builder: $IDBStoreBuilderFunction | $IDBStoreBuilder)
+    {
+        this.storeMap.set(name, isFunction(builder) ? builder(new $IDBStoreBuilder({autoIncrement: false, keyPath: null, indexes: {}, name, schema: null})) : builder);
+        return this as any;
+    }
+
+    /**
+     * Open IDB and initialize, create new IDB if the name of IDB is not exists, or perform the upgrade if version number change.
+     */
+    async open(): Promise<$IDB<Config>> {
+        return new _Promise<$IDB>((resolve, reject) => {
+            const {version: dbVersion, name: dbName, storeMap} = this;
+            const initDBRequest = _indexedDB.open(dbName);
+            const createStoresMap = new Map<string, $IDBStoreBuilder<$IDBStoreConfig>>();
+            const createIndexMap = new Map<$IDBStoreBuilder, Map<string, $IDBIndexConfig>>();
+            const upgradeStoreMap = new Map<string, $IDBStoreBuilder<$IDBStoreConfig>>();
+            const cachedObjectMap = new Map<string, {key: any, value: any}[]>();
+            const unusedStoreNameList: string[] = [];
+            const debug = (message: string) => this.#devMode && console.debug(`[$IDBBuilder (${dbName})]`, message);
+            const storesObject: $IDBConfig['stores'] = _Object_fromEntries(_Array_from(storeMap).map(([name, {config: {keyPath, autoIncrement}, indexes}]) => [
+                name, 
+                {
+                    autoIncrement, keyPath, name,
+                    schema: _null,
+                    indexes: _Object_fromEntries(_Array_from(indexes).map(([name, {keyPath, multiEntry, unique}]) => [
+                        name,
+                        { keyPath, multiEntry, unique } as $IDBIndexConfig
+                    ]))
+                } as $IDBStoreConfig
+            ]))
+            const idbConfig = { version: dbVersion, name: dbName, stores: storesObject };
+            /** IndexedDB initial create function */
+            const initialCreateDB = () => {
+                debug(`No IDB detected, create IDB`);
+                const {transaction, result: idb} = initDBRequest;
+                forEach(storeMap, ([name, storeBuilder]) => {
+                    createStoresMap.set(name, storeBuilder);
+                    createIndexMap.set(storeBuilder, new Map(storeBuilder.indexes))
+                })
+                if (idb.version === dbVersion) upgradeStore(initDBRequest);
+                else transaction!.oncomplete = _ => {
+                    const upgradeDBRequest = indexedDB.open(dbName, dbVersion);
+                    upgradeDBRequest.onupgradeneeded = _ => upgradeStore(upgradeDBRequest);
+                }
+            }
+            /** IndexedDB initial open function */
+            const initialOpenDB = async () => {
+                debug(`IDB Detected`);
+                const idb = initDBRequest.result;
+                const $idb = new $IDB(idb, idbConfig);
+                const transaction = idb[objectStoreNames].length ?  idb.transaction(_Array_from(idb[objectStoreNames]), 'readonly') : null;
+                const noUpgrade = () => {
+                    debug(`No Upgrade`);
+                    resolve($idb);
+                }
+                if (idb.version === dbVersion) return noUpgrade();
+                // get unused stores
+                transaction && forEach(_Array_from(transaction[objectStoreNames]), name => storeMap.has(name) && unusedStoreNameList.push(name))
+                // check store config matches
+                forEach(storeMap, ([storeName, storeBuilder]) => {
+                    const {keyPath, autoIncrement} = storeBuilder.config;
+                    const indexMap = new Map();
+                    const checkIndexes = () =>
+                        forEach(storeBuilder.indexes, ([indexName, indexBuilder]) => {
+                            const [index] = trycatch(() => store?.index(indexName));
+                            const CONFIG_CHANGED = _JSON_stringify(indexBuilder.keyPath) !== _JSON_stringify(index?.keyPath)
+                                || !!indexBuilder.multiEntry !== index?.multiEntry 
+                                || !!indexBuilder.unique !== index?.unique;
+                            if (!index || CONFIG_CHANGED) {
+                                indexMap.set(indexName, indexBuilder);
+                                createIndexMap.set(storeBuilder, indexMap);
+                            }
+                        })
+                    // get store from idb
+                    const [store] = trycatch(() => transaction?.objectStore(storeName));
+                    // create store and break if idb have no store exist
+                    if (!store) return createStoresMap.set(storeName, storeBuilder), checkIndexes();
+                    // define matches variables
+                    const OBJECT_UPGRADE = _Array_from(storeBuilder.upgrades).find(([upgradeVersion],) => 
+                        dbVersion >= upgradeVersion && idb.version < upgradeVersion
+                    )
+                    const CONFIG_CHANGED = 
+                        _JSON_stringify(keyPath) !== _JSON_stringify(store.keyPath)
+                        || autoIncrement !== store?.autoIncrement
+                    const UPGRADE_NEEDED = OBJECT_UPGRADE || CONFIG_CHANGED;
+                    // add indexes
+                    checkIndexes();
+                    // store existed and not need upgrade
+                    if (store && !UPGRADE_NEEDED) return;
+                    // add upgrade store queue
+                    upgradeStoreMap.set(storeName, storeBuilder)
+                    
+                })
+                // resolve if no need upgrade
+                if (dbVersion === idb.version && !createStoresMap.size && !upgradeStoreMap.size && !unusedStoreNameList.length && !createIndexMap.size)
+                    return noUpgrade();
+                // cache objects
+                for (const [storeName, storeBuilder] of upgradeStoreMap) {
+                    const cache: {key: any, value: any}[] = [];
+                    // filter version lower than current idb
+                    const upgradeHandleList = _Array_from(storeBuilder.upgrades)
+                        .filter(([upgradeVersion]) => dbVersion >= upgradeVersion && idb.version < upgradeVersion )
+                        .sort((a, b) => a[0] - b[0])
+                        .map(config => config[1]);
+                    // cache objects from store
+                    await $idb.transaction(storeName, false, async $tx => {
+                        cachedObjectMap.set(storeName, cache);
+                        await $tx.store(storeName).cursor(async cursor => {
+                            cache.push({key: cursor.key, value: cursor.value});
+                            cursor.continue();
+                        })
+                    })
+                    // upgrade objects
+                    for (const upgradeHandle of upgradeHandleList) 
+                        cachedObjectMap.set(storeName, await upgradeHandle(cache, $idb));
+                }
+                // upgrade db from lower version
+                idb.close();
+                const upgradeDBRequest = _indexedDB.open(dbName, dbVersion);
+                upgradeDBRequest[onupgradeneeded] = _ => upgradeStore(upgradeDBRequest);
+                upgradeDBRequest[onsuccess] = _ => {
+                    debug('IDB Upgrade Completed');
+                    resolve(new $IDB(upgradeDBRequest.result, idbConfig));
+                }
+            }
+
+            /** IndexedDB upgrade version */
+            const upgradeStore = (req: IDBOpenDBRequest) => {
+                debug('Upgrade DB')
+                const idb = req.result;
+                /** 'versionchange' type transaction */
+                const transaction = req.transaction as IDBTransaction;
+                // create stores
+                forEach(createStoresMap, ([name, {config}]) => {
+                    idb[createObjectStore](name, config);
+                    debug(`Store Created: ${name}`);
+                });
+                // upgrade stores
+                forEach(upgradeStoreMap, ([name, {config}]) => {
+                    idb[deleteObjectStore](name); 
+                    idb[createObjectStore](name, config);
+                    debug(`Store Upgraded: ${name}`);
+                })
+                // create indexes
+                forEach(createIndexMap, ([{config: {name}}, indexMap]) => {
+                    const store = transaction.objectStore(name);
+                    forEach(indexMap, ([indexName, {keyPath, ...config}]) => {
+                        // if indexes existed, delete and create again
+                        if (store.indexNames.contains(indexName)) store.deleteIndex(indexName);
+                        store.createIndex(indexName, keyPath, config);
+                        debug(`Store '${name}' Index Created: ${indexName}`);
+                    })
+                })
+                // delete unused stores
+                if (this.#deleteUnused) forEach(unusedStoreNameList, name => {
+                    idb[deleteObjectStore](name);
+                    debug(`Unused Store Deleted: ${name}`);
+                });
+                // open db again for insert objects
+                forEach(cachedObjectMap, ([storeName, objectList]) => {
+                    const store = transaction.objectStore(storeName);
+                    forEach(objectList, ({key, value}) => {
+                        if (store.autoIncrement || store.keyPath) store.add(value);
+                        else store.add(value, key)
+                    })
+                    debug(`Recovered Store Objects: ${objectList.length} objects of store '${storeName}'`);
+                })
+            }
+
+            // If db not exist, create db will trigger upgraedneeded event
+            initDBRequest[onupgradeneeded] = initialCreateDB;
+            // If db exist, trigger success event
+            initDBRequest[onsuccess] = initialOpenDB;
+            initDBRequest.onerror = _ => reject(initDBRequest.error);
+        })
+    }
+}
+
+export type $IDBStoreBuilderFunction = (store: $IDBStoreBuilder) => $IDBStoreBuilder<any>;

package/ext/idb/src/structure/builder/$IDBStoreBuilder.ts

@@ -0,0 +1,100 @@
+import type { $IDB } from "#structure/$IDB";
+import { _null } from "amateras/lib/native";
+import type { $IDBStoreConfig } from "#structure/$IDBStore";
+import type { $IDBIndexConfig } from "#structure/$IDBIndex";
+
+export class $IDBStoreBuilder<Config extends $IDBStoreConfig = { name: string, keyPath: null, autoIncrement: false, schema: any, indexes: {} }> {
+    readonly config: Config
+    upgrades = new Map<number, $IDBStoreUpgradeFunction>();
+    indexes = new Map<string, $IDBIndexConfig>();
+    constructor(config: Config) {
+        this.config = config;
+    }
+
+    /**
+     * Define the `keyPath` option of store.
+     * 
+     * [MDN Reference](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API/Using_IndexedDB#structuring_the_database).
+     */
+    keyPath<K extends string[]>(keyPath: K): $IDBStoreBuilder<Prettify<Omit<Config, 'keyPath'> & { keyPath: K }>>;
+    keyPath<K extends string>(keyPath: K): $IDBStoreBuilder<Prettify<Omit<Config, 'keyPath'> & { keyPath: K }>>;
+    keyPath(keyPath: string | string[]) {
+        this.config.keyPath = keyPath;
+        return this as any;
+    }
+
+    /**
+     * Define the `autoIncrement` option of store. 
+     * 
+     * [MDN Reference](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API/Using_IndexedDB#structuring_the_database).
+     */
+    autoIncrement<K extends boolean>(enable: K): $IDBStoreBuilder<Prettify<Omit<Config, 'autoIncrement'> & { autoIncrement: K }>>;
+    autoIncrement(enable: boolean) {
+        this.config.autoIncrement = enable;
+        return this as any;
+    }
+
+    /**
+     * Use the generic type to define store object type.
+     *
+     * @example
+     * store.schema<{
+     *     id: number,
+     *     name: string,
+     *     created: Date,
+     *     updated: Date
+     * }>()
+     */
+    schema<T extends $IDBStoreBuilderSchema<Config>>(): $IDBStoreBuilder<Prettify<Omit<Config, 'schema'> & { schema: T }>>;
+    schema() { return this as any }
+
+    /**
+     * Add new index to store builder.
+     * @param name - Index name
+     * @param config - {@link $IDBIndexOptionalConfig}
+     */
+    index<N extends string, C extends $IDBIndexOptionalConfig<Config>>(name: N, config: C): $IDBStoreBuilder<Prettify<Config & { indexes: Config['indexes'] & Prettify<Record<N, Prettify<$IDBIndexOptionalHandle<N, C>>>> }>>
+    index<C extends $IDBIndexConfig>(name: string, config: C) {
+        this.indexes.set(name, {
+            ...config, name,
+            multiEntry: config.multiEntry ?? false,
+            unique: config.unique ?? false
+        });
+        return this as any;
+    }
+
+    /**
+     * Add store upgrade function to store builder. The store upgrade function is used for change object structure when the store is upgrading. 
+     * @param version - Target version of database
+     * @param handle - Upgrade handle function
+     */
+    upgrade(version: number, handle: $IDBStoreUpgradeFunction) {
+        this.upgrades.set(version, handle);
+        return this;
+    }
+
+}
+
+export type $IDBStoreUpgradeFunction = (objects: {key: IDBValidKey, value: any}[], idb: $IDB<any>) => OrPromise<{key: IDBValidKey, value: any}[]>;
+
+export type $IDBIndexOptionalConfig<StoreConfig extends $IDBStoreConfig = any> = { 
+    keyPath: OrArray<keyof StoreConfig['schema']>, 
+    unique?: boolean, 
+    multiEntry?: boolean 
+}
+
+type $IDBIndexOptionalHandle<N extends string, Config extends $IDBIndexOptionalConfig> = {
+    name: N;
+    keyPath: Config['keyPath'];
+    multiEntry: Config['multiEntry'] extends boolean ? Config['multiEntry'] : false;
+    unique: Config['unique'] extends boolean ? Config['unique'] : false;
+}
+
+type $IDBStoreBuilderSchema<Config extends $IDBStoreConfig> = 
+    Config['keyPath'] extends string
+        ?   Config['autoIncrement'] extends true
+            ?   { [key in Config['keyPath']]: number }
+            :   { [key in Config['keyPath']]: IDBValidKey } 
+        : Config['keyPath'] extends string[]
+            ? { [key in Config['keyPath'][number]]: IDBValidKey }
+            : any;

package/ext/markdown/README.md

@@ -0,0 +1,53 @@
+# amateras/markdown
+
+## Usage
+```ts
+import { Markdown } from 'amateras/markdown';
+
+const markdown = new Markdown();
+
+markdown.parseHTML('# Title'); // => <h1>Title</h1>
+```
+
+## Add Custom Markdown Rules
+```ts
+markdown.lexer.blockTokenizers.set('CUSTOM_TYPE', {
+	regex: /#! (.+)/,
+	handle: matches => {
+		content: lexer.inlineTokenize(matches[1]!)
+	}
+})
+
+markdown.parser.processors.set('CUSTOM_TYPE', token => {
+	return `<custom>${markdown.parser.parse(token.content!)}</custom>`
+})
+```
+
+## Import Syntax
+```ts
+import { MarkdownParser, MarkdownLexer } from 'amateras/markdown';
+import { blockquoteProcessor, blockquoteTokenizer } from 'amateras/markdown/syntax/blockquote';
+import { headingProcessor, headingTokenizer } from 'amateras/markdown/syntax/heading';
+
+const lexer = new MarkdownLexer();
+const parser = new MarkdownParser();
+
+lexer.use(
+    blockquoteTokenizer,
+    headingTokenizer
+)
+
+parser.use(
+    blockquoteProcessor,
+    headingProcessor
+)
+
+function parseHTML(str: string) {
+    const tokens = lexer.blockTokenize(str);
+    return parser.parse(tokens);
+}
+
+parseHTML('# Title') // => <h1>Title</h1>
+parseHTML('> This is Blockquote') // => <blockquote>This is Blockquote</blockquote>
+parseHTML('- List') // => - List
+```

package/ext/markdown/package.json

@@ -0,0 +1,15 @@
+{
+    "name": "@amateras/markdown",
+    "peerDependencies": {
+        "amateras": "../../"
+    },
+    "imports": {
+        "#structure/*": "./src/structure/*.ts",
+        "#lib/*": "./src/lib/*.ts"
+    },
+    "exports": {
+        "./structure/*": "./src/structure/*.ts",
+        "./lib/*": "./src/lib/*.ts",
+        "./syntax/*": "./src/syntax/*.ts"
+    }
+}

package/ext/markdown/src/index.ts

@@ -0,0 +1,3 @@
+export * from '#structure/MarkdownLexer';
+export * from '#structure/MarkdownParser';
+export * from "#structure/Markdown";

package/ext/markdown/src/lib/type.ts

@@ -0,0 +1,26 @@
+export const INLINE = 'INLINE';
+export const BLOCK = 'BLOCK';
+export const TEXT = 'TEXT';
+export const IMAGE = 'IMAGE';
+export const LINK = 'LINK';
+export const QUICK_LINK = 'QUICK_LINK';
+export const CODE = 'CODE';
+export const ITALIC = 'ITALIC';
+export const BOLD = 'BOLD';
+export const TEXT_LINE = 'TEXT_LINE';
+export const HEADING = 'HEADING';
+export const CODE_START = 'CODE_START';
+export const CODE_LINE = 'CODE_LINE';
+export const CODE_END = 'CODE_END';
+export const UNORDERED_LIST_ITEM = 'UNORDERED_LIST_ITEM';
+export const ORDERED_LIST_ITEM = 'ORDERED_LIST_ITEM';
+export const BLOCKQUOTE = 'BLOCKQUOTE';
+export const ALERT = 'ALERT';
+export const ALERT_LINE = 'ALERT_LINE';
+export const HORIZONTAL_RULE = 'HORIZONTAL_RULE';
+export const TABLE = 'TABLE';
+export const TABLE_ROW = 'TABLE_ROW';
+export const TABLE_COLUMN = 'TABLE_COLUMN';
+export const EMPTY_LINE = 'EMPTY_LINE'
+export const INLINE_TEXT = "INLINE_TEXT";
+export const INLINE_CONTENT = "INLINE_CONTENT"

package/ext/markdown/src/lib/util.ts

@@ -0,0 +1,21 @@
+import type { BlockTokenizer, InlineTokenizer, MarkdownLexer } from "#structure/MarkdownLexer";
+import type { MarkdownParseProcessor, MarkdownParser } from "#structure/MarkdownParser";
+import { forEach } from "amateras/lib/native";
+
+export const setBlockTokenizer = (lexer: MarkdownLexer, type: string, tokenizer: BlockTokenizer) => lexer.blockTokenizers.set(type, tokenizer);
+export const setInlineTokenizer = (lexer: MarkdownLexer, type: string, tokenizer: InlineTokenizer) => lexer.inlineTokenizers.set(type, tokenizer);
+
+export const setProcessor = (parser: MarkdownParser, type: string, processor: MarkdownParseProcessor) => parser.processors.set(type, processor);
+
+export const htmltag = (tagname: string, content: string) => `<${tagname}>${content}</${tagname}>`
+
+export const htmlEscapeChar = (str: string) => {
+    forEach([
+        ['&', '&amp;'],
+        ['<', '&lt;'],
+        ['>', '&gt;'],
+        ['"', '&quot;'],
+        ["'", '&#39;']
+    ] as [string, string][], group => str = str.replaceAll(...group))
+    return str;
+}

package/ext/markdown/src/structure/Markdown.ts

@@ -0,0 +1,54 @@
+import { alertProcessor, alertTokenizer } from "../syntax/alert";
+import { blockquoteProcessor, blockquoteTokenizer } from "../syntax/blockquote";
+import { boldProcessor, boldTokenizer } from "../syntax/bold";
+import { codeProcessor, codeTokenizer } from "../syntax/code";
+import { codeblockProcessor, codeblockTokenizer } from "../syntax/codeblock";
+import { headingProcessor, headingTokenizer } from "../syntax/heading";
+import { horizontalRuleProcessor, horizontalRuleTokenizer } from "../syntax/horizontalRule";
+import { imageProcessor, imageTokenizer } from "../syntax/image";
+import { italicProcessor, italicTokenizer } from "../syntax/italic";
+import { linkProcessor, linkTokenizer } from "../syntax/link";
+import { listProcessor, listTokenizer } from "../syntax/list";
+import { textLineProcessor, textProcessor } from "../syntax/text";
+import { MarkdownLexer } from "./MarkdownLexer";
+import { MarkdownParser } from "./MarkdownParser";
+
+export class Markdown {
+    lexer = new MarkdownLexer();
+    parser = new MarkdownParser();
+    constructor() {
+        this.lexer.use(
+            headingTokenizer,
+            codeblockTokenizer,
+            listTokenizer,
+            alertTokenizer,
+            blockquoteTokenizer,
+            horizontalRuleTokenizer,
+            imageTokenizer, // image tokenizer must before link
+            linkTokenizer, // link tokenizer must before bold and italic and code
+            codeTokenizer,
+            boldTokenizer,
+            italicTokenizer,
+        )
+        
+        this.parser.use(
+            textProcessor,
+            imageProcessor,
+            linkProcessor,
+            codeProcessor,
+            italicProcessor,
+            boldProcessor,
+            textLineProcessor,
+            headingProcessor,
+            codeblockProcessor,
+            listProcessor,
+            alertProcessor,
+            blockquoteProcessor,
+            horizontalRuleProcessor
+        )
+    }
+
+    parseHTML(str: string) {
+        return this.parser.parse(this.lexer.blockTokenize(str));
+    }
+}

package/ext/markdown/src/structure/MarkdownLexer.ts

@@ -0,0 +1,111 @@
+import { BLOCK, EMPTY_LINE, INLINE_CONTENT, INLINE_TEXT, TEXT_LINE } from "#lib/type";
+import { forEach, isString } from "amateras/lib/native";
+
+export class MarkdownLexer {
+    blockTokenizers = new Map<string, BlockTokenizer>();
+    inlineTokenizers = new Map<string, InlineTokenizer>();
+    
+    blockTokenize(str: string) {
+        const lines = str.split(/\r?\n/);
+        const tokens: BlockToken[] = [];
+        let lineIndex = 0;
+        lineLoop: while (lineIndex < lines.length) {
+            let line = lines[lineIndex];
+            if (line === undefined) throw 'LINE ERROR';
+            let token: BlockToken | undefined;
+            for (const [type, tokenizer] of this.blockTokenizers) {
+                const matched = line.match(tokenizer.regex);
+                if (matched) {
+                    const {content, multiLine, data} = tokenizer.handle(matched, lineIndex, lines);
+                    token = { layout: BLOCK, type, content, data }
+                    if (multiLine) {
+                        tokens.push(token);
+                        tokens.push(...multiLine.tokens)
+                        lineIndex = multiLine.skip;
+                        continue lineLoop;
+                    }
+                    break;
+                }
+            }
+            if (!token) token = { 
+                layout: BLOCK, 
+                ...(
+                    line.length 
+                    ?   { type: TEXT_LINE, content: this.inlineTokenize(line) } 
+                    :   { type: EMPTY_LINE, content: [] }
+                )
+            };
+            tokens.push(token);
+            lineIndex++;
+        }
+        return tokens;
+    }
+
+    inlineTokenize(str: string): InlineToken[] {
+        const tokens: InlineToken[] = [];
+        let remainStr = str;
+        while (remainStr.length) {
+            let token: InlineToken | undefined;
+            for (const [type, tokenizer] of this.inlineTokenizers) {
+                const matched = remainStr.match(tokenizer.regex);
+                if (matched) {
+                    const {index, 0: matchStr} = matched;
+                    // handle before matched string
+                    if (index != 0) tokens.push(...this.inlineTokenize(remainStr.substring(0, index)));
+                    // handle matched string
+                    const {content, data} = tokenizer.handle(matched);
+                    token = { type, ...(isString(content) ? { layout: INLINE_TEXT, text: content } : { layout: INLINE_CONTENT, content })};
+                    if (data) token.data = data;
+                    remainStr = remainStr.substring(index! + matchStr.length);
+                    break;
+                }
+            }
+            if (!token) {
+                token = { type: 'TEXT', layout: INLINE_TEXT, text: remainStr };
+                remainStr = '';
+            }
+            tokens.push(token);
+        }
+        return tokens;
+    }
+
+    use(...handle: ((parser: this) => void)[]) {
+        forEach(handle, fn => fn(this));
+        return this;
+    }
+}
+
+export type BlockTokenizer = {
+    regex: RegExp;
+    handle: (matches: RegExpMatchArray, position: number, lines: string[]) => { content: (InlineToken | BlockToken)[], multiLine?: BlockTokenizerMultiLine, data?: {[key: string]: any} };
+}
+export type BlockTokenizerMultiLine = {
+    skip: number;
+    tokens: BlockToken[];
+}
+export type InlineTokenizer = {
+    regex: RegExp;
+    handle: (matches: RegExpMatchArray) => { content: InlineToken[] | string, data?: {[key: string]: any} }
+}
+
+export interface TokenBase {
+    type: string;
+    layout: 'BLOCK' | 'INLINE_CONTENT' | 'INLINE_TEXT';
+    content?: Token[];
+    text?: string;
+    data?: {[key: string]: any};
+}
+export interface BlockToken extends TokenBase {
+    layout: 'BLOCK'
+    content: Token[];
+}
+export interface InlineTextToken extends TokenBase {
+    layout: 'INLINE_TEXT'
+    text: string;
+}
+export interface InlineContentToken extends TokenBase {
+    layout: 'INLINE_CONTENT'
+    content: (InlineToken)[];
+}
+export type InlineToken = InlineTextToken | InlineContentToken;
+export type Token = BlockToken | InlineToken;

package/ext/markdown/src/structure/MarkdownParser.ts

@@ -0,0 +1,33 @@
+import { forEach, isString } from "../../../../src/lib/native";
+import { type Token } from "./MarkdownLexer";
+
+export class MarkdownParser {
+    processors = new Map<string, MarkdownParseProcessor>();
+    
+    parse(tokens: (Token)[]) {
+        let html = '';
+        let i = 0;
+        while (i < tokens.length) {
+            const token = tokens[i]!;
+            const processor = this.processors.get(token.type);
+            if (processor) {
+                const result = processor(token, tokens.slice(i));
+                if (isString(result)) {
+                    html += result;
+                } else {
+                    html += result.html;
+                    i += result.skipTokens;
+                }
+            }
+            i++;
+        }
+        return html;
+    }
+
+    use(...handle: ((parser: this) => void)[]) {
+        forEach(handle, fn => fn(this));
+        return this;
+    }
+}
+
+export type MarkdownParseProcessor = (token: Token, tokens: Token[]) => (string | { html: string, skipTokens: number })