@kerebron/extension-codemirror 0.3.2 → 0.4.2

package/esm/extension-markdown/src/treeSitterTokenizer.js

@@ -0,0 +1,769 @@
+import { createParser } from '../../tree-sitter-shim/src/main.js';
+import { NESTING_CLOSING, NESTING_OPENING, NESTING_SELF_CLOSING, Token, } from './types.js';
+import { fetchWasm, getLangTreeSitter } from '../../wasm/src/mod.js';
+function treeToTokens(tree, inlineParser) {
+    const retVal = [];
+    let blockLevel = 0;
+    const pushInlineNode = (someInlineToken, debug) => {
+        const lastBlockToken = retVal[retVal.length - 1];
+        if (lastBlockToken?.type === 'inline') {
+            if (!lastBlockToken.children) {
+                lastBlockToken.children = [];
+            }
+            lastBlockToken.children.push(someInlineToken);
+            lastBlockToken.content = someInlineToken.content;
+            return;
+        }
+        const blockToken = new Token('inline', '', NESTING_SELF_CLOSING);
+        blockToken.info = debug;
+        blockToken.level = blockLevel;
+        blockToken.children = [someInlineToken];
+        blockToken.content = blockToken.children
+            .map((c) => c.content || '').join('');
+        retVal.push(blockToken);
+    };
+    const walkInline = (children, startPosition = { row: 0, column: 0 }) => {
+        for (const node of children) {
+            const map = [
+                startPosition.row + node.startPosition?.row,
+                startPosition.row + node.endPosition?.row,
+                startPosition.column + node.startPosition?.column,
+                (+node.endPosition.row === +node.startPosition.row
+                    ? startPosition.column
+                    : 0) + node.endPosition?.column,
+            ];
+            if (node.type.length === 1) { // single letter type is text
+                const token = new Token('text', '', NESTING_SELF_CLOSING);
+                token.map = map;
+                token.meta = 'noEscText';
+                token.content = node.text ?? '';
+                pushInlineNode(token, 'walkInline');
+                continue;
+            }
+            switch (node.type) {
+                case 'block_continuation':
+                case 'emphasis_delimiter':
+                case 'code_span_delimiter':
+                    break;
+                case 'text':
+                    {
+                        const token = new Token('text', '', NESTING_SELF_CLOSING);
+                        token.map = map;
+                        token.meta = 'noEscText';
+                        token.content = node.text ?? '';
+                        pushInlineNode(token, 'text');
+                        break;
+                    }
+                    break;
+                case 'latex_block':
+                    {
+                        const delimiter = node.children
+                            .find((c) => c.type === 'latex_span_delimiter');
+                        const content = node.children
+                            .filter((c) => c.type !== 'latex_span_delimiter')
+                            .map((c) => c.text)
+                            .join('');
+                        if (delimiter?.text === '$$') {
+                            const token = new Token('fence', 'pre', NESTING_SELF_CLOSING);
+                            token.level = blockLevel;
+                            token.markup = '$$';
+                            token.info = 'latex';
+                            token.content = content.trim();
+                            retVal.push(token);
+                        }
+                        else {
+                            const token = new Token('math', '', NESTING_SELF_CLOSING);
+                            token.map = map;
+                            token.meta = 'noEscText';
+                            token.content = content;
+                            pushInlineNode(token, 'latex_block');
+                        }
+                    }
+                    break;
+                case 'whitespace':
+                    if (node.text === '\n') {
+                        const token = new Token('hardbreak', 'br', NESTING_SELF_CLOSING);
+                        token.map = map;
+                        token.content = node.text ?? '';
+                        pushInlineNode(token, 'whitespace \n');
+                        break;
+                    }
+                    if (node.text && node.text.match(/^\s+$/)) {
+                        const token = new Token('text', 'br', NESTING_SELF_CLOSING);
+                        token.map = map;
+                        token.meta = 'noEscText';
+                        token.content = node.text ?? '';
+                        pushInlineNode(token, 'whitespace space');
+                        break;
+                    }
+                    if (node.text.trim().length > 0) {
+                        const token = new Token('text', '', NESTING_SELF_CLOSING);
+                        token.map = map;
+                        token.meta = 'noEscText';
+                        token.content = node.text ?? '';
+                        pushInlineNode(token, 'whitespace other');
+                        break;
+                    }
+                    break;
+                case 'shortcut_link':
+                    {
+                        const tokenName = 'link';
+                        const tagName = 'a';
+                        const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                        const destination = node.children
+                            .find((c) => c.type === 'link_destination');
+                        openToken.attrSet('href', destination?.text || '');
+                        const title = node.children
+                            .find((c) => c.type === 'link_title');
+                        if (title) {
+                            openToken.attrSet('title', title.text || '');
+                        }
+                        pushInlineNode(openToken, 'shortcut_link');
+                        node.children
+                            .filter((c) => c.type === 'link_text')
+                            .forEach((c) => {
+                            const token = new Token('text', '', NESTING_SELF_CLOSING);
+                            token.map = map;
+                            token.meta = 'noEscText';
+                            token.content = c.text ?? '';
+                            pushInlineNode(token, 'link_text');
+                        });
+                        // walkInline(node.children);
+                        const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                        pushInlineNode(closeToken, '/shortcut_link');
+                    }
+                    break;
+                case 'inline_link':
+                    {
+                        const tokenName = 'link';
+                        const tagName = 'a';
+                        const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                        const destination = node.children
+                            .find((c) => c.type === 'link_destination');
+                        openToken.attrSet('href', destination?.text || '');
+                        const title = node.children
+                            .find((c) => c.type === 'link_title');
+                        if (title) {
+                            openToken.attrSet('title', title.text || '');
+                        }
+                        pushInlineNode(openToken, 'inline_link');
+                        node.children
+                            .filter((c) => c.type === 'link_text')
+                            .forEach((c) => {
+                            const token = new Token('text', '', NESTING_SELF_CLOSING);
+                            token.map = map;
+                            token.meta = 'noEscText';
+                            token.content = c.text ?? '';
+                            pushInlineNode(token, 'inline_link_txt');
+                        });
+                        // walkInline(node.children);
+                        const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                        pushInlineNode(closeToken, '/inline_link');
+                    }
+                    break;
+                case 'image':
+                    {
+                        const tokenName = 'image';
+                        const tagName = 'img';
+                        const token = new Token(tokenName, tagName, NESTING_SELF_CLOSING);
+                        const image_description = node.children
+                            .find((c) => c.type === 'image_description');
+                        token.attrSet('alt', image_description?.text || '');
+                        const destination = node.children
+                            .find((c) => c.type === 'link_destination');
+                        token.attrSet('src', destination?.text || '');
+                        const title = node.children
+                            .find((c) => c.type === 'link_title');
+                        if (title) {
+                            token.attrSet('title', title.text || '');
+                        }
+                        pushInlineNode(token, 'image');
+                    }
+                    break;
+                case 'strong_emphasis':
+                    {
+                        const delimiter = node.children
+                            .find((c) => c.type === 'emphasis_delimiter');
+                        const tokenName = delimiter?.text === '_' ? 'underline' : 'strong';
+                        const tagName = delimiter?.text === '_' ? 'u' : 'strong';
+                        const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                        if (delimiter?.text === '_') {
+                            openToken.markup = '__';
+                        }
+                        pushInlineNode(openToken, 'strongem');
+                        walkInline(node.children);
+                        const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                        if (delimiter?.text === '_') {
+                            closeToken.markup = '__';
+                        }
+                        pushInlineNode(closeToken, '/strongem');
+                    }
+                    break;
+                case 'emphasis':
+                    {
+                        const delimiter = node.children
+                            .find((c) => c.type === 'emphasis_delimiter');
+                        const children = node.children
+                            .filter((c) => c.type !== 'emphasis_delimiter');
+                        const tokenName = delimiter?.text === '_' ? 'underline' : 'em';
+                        const tagName = delimiter?.text === '_' ? 'u' : 'em';
+                        const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                        pushInlineNode(openToken, 'em');
+                        walkInline(children);
+                        const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                        pushInlineNode(closeToken, '/em');
+                    }
+                    break;
+                case 'strikethrough':
+                    {
+                        const tokenName = 'strike';
+                        const tagName = 'strike';
+                        const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                        pushInlineNode(openToken, 's');
+                        walkInline(node.children);
+                        const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                        pushInlineNode(closeToken, '/s');
+                    }
+                    break;
+                case 'code_span':
+                    {
+                        const tokenName = 'code';
+                        const tagName = 'code';
+                        const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                        pushInlineNode(openToken, 'code');
+                        walkInline(node.children);
+                        const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                        pushInlineNode(closeToken, '/code');
+                    }
+                    break;
+                case 'html_tag':
+                    {
+                        const tokenName = 'html_block';
+                        const tagName = '';
+                        const token = new Token(tokenName, tagName, NESTING_SELF_CLOSING);
+                        token.content = node.text;
+                        pushInlineNode(token, 'html_block');
+                    }
+                    break;
+                default:
+                    console.debug('inline_node', node);
+                    throw new Error(`Unhandled inline node type: ${node.type}`);
+            }
+        }
+    };
+    const walkRecursive = (node, ctx = { tableRowType: 'tbody', cellNo: 0, cellAlign: [] }) => {
+        if (!node) {
+            return;
+        }
+        if (node.type === 'inline') {
+            const inlineText = node.text;
+            if (!inlineText) {
+                throw new Error('!inlineText');
+            }
+            if (node.children.length > 0) {
+                walkInline(node.children, node.startPosition);
+            }
+            else {
+                const inlineTree = inlineParser.parse(inlineText);
+                if (!inlineTree) {
+                    throw new Error('!inlineTree');
+                }
+                walkInline(inlineTree?.rootNode.children, node.startPosition);
+            }
+            return;
+        }
+        if (!node.children || node.children.length === 0) {
+            if (node.type === 'text' || node.type.length === 1) {
+                const token = new Token('text', '', NESTING_SELF_CLOSING);
+                token.map = [
+                    +node.startPosition?.row,
+                    +node.endPosition?.row,
+                    +node.startPosition?.column,
+                    +node.endPosition?.column,
+                ];
+                token.meta = 'noEscText';
+                token.content = node.text ?? '';
+                pushInlineNode(token, 'txt2');
+                return;
+            }
+            if (node.type === 'whitespace') {
+                if (blockLevel === 0) {
+                    return;
+                }
+                if (node.text.match(/^\n+$/)) {
+                    //   const token = new Token('hardbreak', 'br', NESTING_SELF_CLOSING);
+                    // token.map = [ +node.startPosition?.row, +node.endPosition?.row, +node.startPosition?.column, +node.endPosition?.column ];
+                    //   token.content = node.text ?? '';
+                    //   retVal.push(token);
+                    return;
+                }
+                if (node.text && node.text.match(/^\s+$/)) {
+                    const token = new Token('text', 'br', NESTING_SELF_CLOSING);
+                    token.map = [
+                        +node.startPosition?.row,
+                        +node.endPosition?.row,
+                        +node.startPosition?.column,
+                        +node.endPosition?.column,
+                    ];
+                    token.meta = 'noEscText';
+                    token.content = node.text ?? '';
+                    pushInlineNode(token, 'txt3');
+                    return;
+                }
+                if (node.text.trim().length > 0) {
+                    const token = new Token('text', '', NESTING_SELF_CLOSING);
+                    token.map = [
+                        +node.startPosition?.row,
+                        +node.endPosition?.row,
+                        +node.startPosition?.column,
+                        +node.endPosition?.column,
+                    ];
+                    token.meta = 'noEscText';
+                    token.content = node.text ?? '';
+                    pushInlineNode(token, 'txt4');
+                    return;
+                }
+            }
+        }
+        switch (node.type) {
+            case 'block_continuation':
+                break;
+            case 'document':
+            case 'section':
+                node.children?.forEach((child) => walkRecursive(child, ctx));
+                break;
+            case 'thematic_break':
+                {
+                    const token = new Token('hr', 'hr', NESTING_SELF_CLOSING);
+                    token.level = blockLevel;
+                    token.map = [
+                        +node.startPosition?.row,
+                        +node.endPosition?.row,
+                        +node.startPosition?.column,
+                        +node.endPosition?.column,
+                    ];
+                    token.markup = node.text;
+                    retVal.push(token);
+                }
+                break;
+            case 'setext_heading':
+                // case 'setext_h2_underline':
+                // TODO
+                // TODO
+                {
+                    const tokenName = 'heading';
+                    const underline = node.children.find((child) => child.type.startsWith('setext_') &&
+                        child.type.endsWith('_underline'));
+                    // const marker = node.children.find((child) =>
+                    //   child.type.startsWith('atx_') && child.type.endsWith('_marker')
+                    // );
+                    const tagName = underline?.type.substring('setext_'.length, 'setext_'.length + 2) ||
+                        'h1';
+                    const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                    openToken.level = blockLevel;
+                    openToken.markup = underline?.text || '#';
+                    retVal.push(openToken);
+                    const children = node.children.filter((c) => !c.type.startsWith('setext_'));
+                    blockLevel++;
+                    children.forEach((child) => walkRecursive(child, ctx));
+                    blockLevel--;
+                    const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                    closeToken.level = blockLevel;
+                    closeToken.markup = underline?.text || '';
+                    retVal.push(closeToken);
+                }
+                break;
+            case 'atx_heading':
+                {
+                    const tokenName = 'heading';
+                    const marker = node.children.find((child) => child.type.startsWith('atx_') && child.type.endsWith('_marker'));
+                    const tagName = marker?.type.substring('atx_'.length, 'atx_'.length + 2) || 'h1';
+                    const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                    openToken.level = blockLevel;
+                    openToken.markup = marker?.text || '#';
+                    retVal.push(openToken);
+                    const children = [...node.children];
+                    if (children.length > 0 && children[0].type.endsWith('_marker')) {
+                        children.splice(0, 1);
+                    }
+                    if (children.length > 0 && children[0].type === 'whitespace') { // ' '
+                        children.splice(0, 1);
+                    }
+                    if (children.length > 0 &&
+                        children[children.length - 1].type === 'whitespace') { // /n
+                        children.splice(children.length - 1, 1);
+                    }
+                    blockLevel++;
+                    children.forEach((child) => walkRecursive(child, ctx));
+                    blockLevel--;
+                    const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                    closeToken.level = blockLevel;
+                    retVal.push(closeToken);
+                }
+                break;
+            case 'minus_metadata':
+                {
+                    const tokenName = 'frontmatter';
+                    const tagName = 'frontmatter';
+                    const token = new Token(tokenName, tagName, NESTING_SELF_CLOSING);
+                    token.level = blockLevel;
+                    token.content = node.text || '';
+                    retVal.push(token);
+                }
+                break;
+            case 'fenced_code_block':
+                {
+                    const tokenName = 'fence';
+                    const tagName = 'pre';
+                    const info = node.children.find((child) => child.type === 'info_string');
+                    const token = new Token(tokenName, tagName, NESTING_SELF_CLOSING);
+                    token.level = blockLevel;
+                    token.markup = '```';
+                    if (info) {
+                        token.attrSet('lang', info.text);
+                    }
+                    const children = [...node.children
+                            .filter((item) => item.type === 'code_fence_content')];
+                    const content = children.map((item) => item.children
+                        .map((inline) => inline.text || '')
+                        .join('')).join('');
+                    token.content = content;
+                    retVal.push(token);
+                }
+                break;
+            case 'block_quote':
+                {
+                    const tokenName = 'blockquote';
+                    const tagName = 'blockquote';
+                    const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                    openToken.level = blockLevel;
+                    retVal.push(openToken);
+                    blockLevel++;
+                    node.children
+                        ?.filter((child) => !['block_quote_marker'].includes(child.type))
+                        .forEach((child) => walkRecursive(child, ctx));
+                    blockLevel--;
+                    const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                    closeToken.level = blockLevel;
+                    retVal.push(closeToken);
+                }
+                break;
+            case 'indented_code_block':
+                {
+                    const tokenName = 'code_block';
+                    const tagName = 'code_block';
+                    const token = new Token(tokenName, tagName, NESTING_SELF_CLOSING);
+                    token.level = blockLevel;
+                    let indent = 0;
+                    if ('indent' in node) {
+                        indent = node.indent.length;
+                    }
+                    const lines = node.text.split('\n');
+                    for (const line of lines) {
+                        if (line.trim().length === 0) {
+                            continue;
+                        }
+                        const m = line.match(/^ +/);
+                        if (!m) {
+                            indent = 0;
+                        }
+                        else {
+                            if (indent > m[0].length) {
+                                indent = m[0].length;
+                            }
+                        }
+                    }
+                    const content = lines
+                        .map((line) => line.substring(indent))
+                        .join('\n');
+                    token.attrSet('indent', '' + indent);
+                    token.content = content;
+                    retVal.push(token);
+                }
+                break;
+            case 'paragraph':
+                {
+                    const tokenName = 'paragraph';
+                    const tagName = 'p';
+                    const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                    openToken.level = blockLevel;
+                    retVal.push(openToken);
+                    blockLevel++;
+                    node.children?.forEach((child) => walkRecursive(child, ctx));
+                    blockLevel--;
+                    const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                    closeToken.level = blockLevel;
+                    retVal.push(closeToken);
+                }
+                break;
+            case 'pipe_table_cell':
+                {
+                    const tokenName = ctx.tableRowType === 'thead' ? 'th' : 'td';
+                    const tagName = ctx.tableRowType === 'thead' ? 'th' : 'td';
+                    const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                    openToken.attrSet('align', ctx.cellAlign[ctx.cellNo] || 'left');
+                    openToken.level = blockLevel;
+                    retVal.push(openToken);
+                    blockLevel++;
+                    {
+                        const tokenName = 'paragraph';
+                        const tagName = 'p';
+                        const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                        openToken.level = blockLevel;
+                        retVal.push(openToken);
+                        const children = [...node.children];
+                        while (children.length > 0) {
+                            if (children[children.length - 1].type === 'whitespace') {
+                                children.splice(children.length - 1, 1);
+                            }
+                            else {
+                                break;
+                            }
+                        }
+                        if (children.length > 0) {
+                            children
+                                .forEach((child) => walkRecursive(child, ctx));
+                        }
+                        else {
+                            const map = [
+                                node.startPosition?.row,
+                                node.endPosition?.row,
+                                node.startPosition?.column,
+                                node.endPosition?.column,
+                            ];
+                            const token = new Token('text', '', NESTING_SELF_CLOSING);
+                            token.map = map;
+                            token.meta = 'noEscText';
+                            token.content = node.text?.replace(/\s+$/, '') ?? '';
+                            pushInlineNode(token, 'txt5');
+                        }
+                        const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                        closeToken.level = blockLevel;
+                        retVal.push(closeToken);
+                    }
+                    blockLevel--;
+                    const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                    closeToken.level = blockLevel;
+                    retVal.push(closeToken);
+                    ctx.cellNo++;
+                }
+                break;
+            case 'pipe_table_header':
+            case 'pipe_table_row':
+                {
+                    const tokenName = 'tr';
+                    const tagName = 'tr';
+                    const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                    openToken.level = blockLevel;
+                    retVal.push(openToken);
+                    blockLevel++;
+                    node.children
+                        ?.filter((c) => c.type === 'pipe_table_cell')
+                        .forEach((child) => walkRecursive(child, ctx));
+                    blockLevel--;
+                    const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                    closeToken.level = blockLevel;
+                    retVal.push(closeToken);
+                }
+                break;
+            case 'pipe_table':
+                {
+                    const tokenName = 'table';
+                    const tagName = 'table';
+                    const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                    openToken.level = blockLevel;
+                    retVal.push(openToken);
+                    blockLevel++;
+                    const delimiterRows = node.children
+                        ?.filter((c) => c.type === 'pipe_table_delimiter_row');
+                    const cellAlign = [];
+                    if (delimiterRows.length > 0) {
+                        const delimiterRow = delimiterRows[0];
+                        const delimiterCells = delimiterRow.children
+                            ?.filter((c) => c.type === 'pipe_table_delimiter_cell');
+                        for (let cellNo = 0; cellNo < delimiterCells.length; cellNo++) {
+                            const cell = delimiterCells[cellNo];
+                            if (cell.children.find((c) => c.type === 'pipe_table_align_right')) {
+                                cellAlign.push('right');
+                            }
+                            else {
+                                cellAlign.push('left');
+                            }
+                        }
+                    }
+                    const headRows = node.children
+                        ?.filter((c) => c.type === 'pipe_table_header');
+                    if (headRows.length > 0) {
+                        const tokenName = 'thead';
+                        const tagName = 'thead';
+                        const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                        openToken.level = blockLevel;
+                        retVal.push(openToken);
+                        blockLevel++;
+                        headRows.forEach((child) => walkRecursive(child, {
+                            ...ctx,
+                            tableRowType: 'thead',
+                            cellNo: 0,
+                            cellAlign: cellAlign,
+                        }));
+                        blockLevel--;
+                        const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                        closeToken.level = blockLevel;
+                        retVal.push(closeToken);
+                    }
+                    const bodyRows = node.children
+                        ?.filter((c) => c.type === 'pipe_table_row');
+                    if (bodyRows.length > 0) {
+                        // const tokenName = 'tbody';
+                        // const tagName = 'tbody';
+                        // const openToken = new Token(
+                        //   tokenName + '_open',
+                        //   tagName,
+                        //   NESTING_OPENING,
+                        // );
+                        // openToken.level = blockLevel;
+                        // retVal.push(openToken);
+                        blockLevel++;
+                        bodyRows.forEach((child) => walkRecursive(child, {
+                            ...ctx,
+                            tableRowType: 'tbody',
+                            cellAlign: cellAlign,
+                        }));
+                        blockLevel--;
+                        // const closeToken = new Token(
+                        //   tokenName + '_close',
+                        //   tagName,
+                        //   NESTING_CLOSING,
+                        // );
+                        // closeToken.level = blockLevel;
+                        // retVal.push(closeToken);
+                    }
+                    blockLevel--;
+                    const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                    closeToken.level = blockLevel;
+                    retVal.push(closeToken);
+                }
+                break;
+            case 'list':
+                {
+                    let tokenName = 'bullet_list';
+                    let tagName = 'ul';
+                    let start = '';
+                    const firstItem = node.children.find((item) => item.type === 'list_item');
+                    if (firstItem) {
+                        const taskListMarker = firstItem.children.find((item) => item.type.startsWith('task_list_marker'));
+                        const listMarker = firstItem.children.find((item) => item.type.startsWith('list_marker_dot'));
+                        if (taskListMarker) {
+                            tokenName = 'task_list';
+                            tagName = 'ul';
+                        }
+                        else if (listMarker) {
+                            tokenName = 'ordered_list';
+                            tagName = 'ol';
+                            start = listMarker.text.trim().replace('.', '');
+                        }
+                    }
+                    const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                    openToken.level = blockLevel;
+                    openToken.markup = tagName === 'ol' ? '.' : '*';
+                    openToken.attrSet('symbol', tagName === 'ol' ? '1' : '*');
+                    if (start) {
+                        openToken.attrSet('start', start);
+                    }
+                    retVal.push(openToken);
+                    blockLevel++;
+                    node.children?.forEach((child) => walkRecursive(child, ctx));
+                    blockLevel--;
+                    const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                    closeToken.level = blockLevel;
+                    retVal.push(closeToken);
+                }
+                break;
+            case 'list_item':
+                {
+                    const taskListMarker = node.children.find((item) => item.type.startsWith('task_list_marker_'));
+                    const listMarker = node.children.find((item) => item.type.startsWith('list_marker_'));
+                    if (!listMarker) {
+                        break;
+                    }
+                    let tokenName = taskListMarker ? 'task_item' : 'list_item';
+                    const tagName = 'li';
+                    const openToken = new Token(tokenName + '_open', tagName, NESTING_OPENING);
+                    openToken.level = blockLevel;
+                    openToken.markup = listMarker.text.trim();
+                    if (listMarker.type === 'list_marker_dot') {
+                        // openToken.info = listMarker.text.trim().replace('.', '');
+                        // openToken.markup = '.';
+                        openToken.info = '';
+                        openToken.markup = '';
+                    }
+                    if (taskListMarker) {
+                        if (taskListMarker.type === 'task_list_marker_checked') {
+                            // throw new Error('aaaa')
+                        }
+                        openToken.attrSet('checked', taskListMarker.type === 'task_list_marker_checked'
+                            ? 'checked'
+                            : '');
+                    }
+                    retVal.push(openToken);
+                    // node.children?.forEach((child) => walkRecursive(child, ctx));
+                    blockLevel++;
+                    walkRecursive(node.children.find((item) => item.type === 'paragraph'), ctx);
+                    const lists = node.children.filter((item) => item.type === 'list');
+                    for (const list of lists) {
+                        walkRecursive(list, ctx);
+                    }
+                    blockLevel--;
+                    const closeToken = new Token(tokenName + '_close', tagName, NESTING_CLOSING);
+                    closeToken.level = blockLevel;
+                    retVal.push(closeToken);
+                }
+                break;
+            case 'html_block':
+                {
+                    const token = new Token('html_block', '', NESTING_SELF_CLOSING);
+                    token.level = blockLevel;
+                    token.content = node.text ?? '';
+                    retVal.push(token);
+                }
+                break;
+            case 'text':
+                {
+                    const token = new Token('text', '', NESTING_SELF_CLOSING);
+                    token.level = blockLevel;
+                    token.content = node.text ?? '';
+                    retVal.push(token);
+                }
+                break;
+            default:
+                // Log unhandled node types for debugging
+                console.warn(`Unhandled node type: ${node.type}, children: ${node.children.map((c) => c.type).join(', ')}`, {
+                    ...node,
+                    children: undefined,
+                    _children: undefined,
+                    tree: undefined,
+                });
+                throw new Error('aaa');
+        }
+    };
+    walkRecursive(tree.rootNode);
+    return retVal;
+}
+export async function sitterTokenizer(cdnUrl = '') {
+    const jsonManifest = getLangTreeSitter('markdown', cdnUrl);
+    const blockUrl = jsonManifest.files.find((url) => url.indexOf('_inline') === -1);
+    const inlineUrl = jsonManifest.files.find((url) => url.indexOf('_inline') > -1);
+    const markdownWasm = await fetchWasm(blockUrl);
+    const inlineWasm = await fetchWasm(inlineUrl);
+    const blockParser = (await createParser(markdownWasm));
+    const inlineParser = (await createParser(inlineWasm));
+    return {
+        parse: (source) => {
+            const tree = blockParser.parse(source);
+            if (!tree) {
+                throw new Error('Tree is null');
+            }
+            return treeToTokens(tree, inlineParser);
+        },
+    };
+}