@kerebron/extension-lsp 0.4.1

package/esm/extension-markdown/src/pmToMdConverter.js

@@ -0,0 +1,433 @@
+import { Fragment } from 'prosemirror-model';
+import { Token } from './types.js';
+import { nodeToTreeStringOutput, } from '../../editor/src/mod.js';
+import { MarkdownSerializer } from './MarkdownSerializer.js';
+import { DocumentMarkdownTokenizer } from './DocumentMarkdownTokenizer.js';
+import { SmartOutput } from '../../editor/src/utilities/mod.js';
+function removeMarkedContent(node, markType) {
+    if (node.isText) {
+        const hasMark = node.marks.some((mark) => mark.type === markType);
+        return hasMark ? null : node;
+    }
+    if (node.isLeaf) {
+        return node;
+    }
+    const newContent = [];
+    node.content.forEach((child) => {
+        const cleaned = removeMarkedContent(child, markType);
+        if (cleaned) {
+            newContent.push(cleaned);
+        }
+    });
+    return node.copy(Fragment.fromArray(newContent));
+}
+class MdStashContext {
+    constructor() {
+        Object.defineProperty(this, "ctxStash", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "currentCtx", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.currentCtx = {
+            meta: {},
+        };
+        this.stash();
+    }
+    stash() {
+        this.ctxStash.push(this.currentCtx);
+        this.currentCtx = {
+            ...structuredClone({ ...this.currentCtx }),
+        };
+        return this.ctxStash.length - 1;
+    }
+    unstash() {
+        const ctx = this.ctxStash.pop();
+        if (!ctx) {
+            throw new Error('Unstash failed');
+        }
+        this.currentCtx = ctx;
+    }
+    get current() {
+        return this.currentCtx;
+    }
+}
+export async function pmToMdConverter(document, config, schema, editor) {
+    const result = syncPmToMdConverter(document, config, schema, editor);
+    return new TextEncoder().encode(result.content);
+}
+export function syncPmToMdConverter(document, config, schema, editor) {
+    const ctx = new MdStashContext();
+    // TODO: refactor to Tokenizer
+    const defaultMarkdownTokenizer = new DocumentMarkdownTokenizer({
+        paragraph(node) {
+            return {
+                open: 'paragraph_open',
+                close: 'paragraph_close',
+            };
+        },
+        hr(node) {
+            return {
+                selfClose: 'hr',
+            };
+        },
+        heading(node) {
+            return {
+                open: (node) => {
+                    if (!node.attrs.level) {
+                        throw new Error('No heading level');
+                    }
+                    const token = new Token('heading_open', 'h' + node.attrs.level, 1);
+                    return token;
+                },
+                ///////
+                close: 'heading_close',
+            };
+            // state.write(state.repeat('#', node.attrs.level) + ' ');
+            // state.renderInline(node, false);
+            // state.closeBlock(node);
+        },
+        blockquote(node) {
+            return {
+                open: 'blockquote_open',
+                close: 'blockquote_close',
+            };
+        },
+        math() {
+            return {
+                open: 'math_block_open',
+                close: 'math_block_close',
+                // math_inline
+                // selfClose: (node) => {
+                //   const token = new Token('math', 'math', 0);
+                //   token.attrSet('content', node.attrs.content);
+                //   if (node.attrs.type) {
+                //   }
+                //   return token;
+                // },
+            };
+        },
+        task_list(node) {
+            return {
+                open: (node) => {
+                    ctx.stash();
+                    ctx.current.meta['list_type'] = 'tl';
+                    ctx.current.meta['list_type_symbol'] = '';
+                    const token = new Token('task_list_open', 'ul', 1);
+                    token.attrSet('symbol', '*');
+                    return token;
+                },
+                close: (node) => {
+                    ctx.unstash();
+                    return new Token('task_list_close', 'ul', -1);
+                },
+            };
+        },
+        bullet_list(node) {
+            return {
+                open: (node) => {
+                    ctx.stash();
+                    ctx.current.meta['list_type'] = 'ul';
+                    ctx.current.meta['list_type_symbol'] = '*';
+                    const token = new Token('bullet_list_open', 'ul', 1);
+                    token.attrSet('symbol', '*');
+                    return token;
+                },
+                close: (node) => {
+                    ctx.unstash();
+                    return new Token('bullet_list_close', 'ul', -1);
+                },
+            };
+        },
+        ordered_list(node) {
+            return {
+                open: (node) => {
+                    ctx.stash();
+                    ctx.current.meta['list_type'] = 'ol';
+                    ctx.current.meta['list_type_symbol'] = node.attrs['type'] ||
+                        '1';
+                    const token = new Token('ordered_list_open', 'ol', 1);
+                    token.attrSet('symbol', node.attrs['type'] || '1');
+                    return token;
+                },
+                close: (node) => {
+                    ctx.unstash();
+                    return new Token('ordered_list_close', 'ol', -1);
+                },
+            };
+        },
+        list_item(node) {
+            return {
+                open: (node) => {
+                    const token = new Token('list_item_open', 'li', 1);
+                    if (ctx.current.meta['list_type'] === 'ul') {
+                        token.markup = ctx.current.meta['list_type_symbol'] || '-';
+                    }
+                    return token;
+                },
+                close: 'list_item_close',
+            };
+        },
+        task_item(node) {
+            return {
+                open: (node) => {
+                    const token = new Token('task_item_open', 'li', 1);
+                    token.attrSet('checked', node.attrs.checked ? 'checked' : '');
+                    return token;
+                },
+                close: 'task_item_close',
+            };
+        },
+        table(node) {
+            return {
+                open: 'table_open',
+                close: 'table_close',
+            };
+        },
+        table_row(node) {
+            return {
+                open: 'tr_open',
+                close: 'tr_close',
+            };
+        },
+        table_cell(node) {
+            return {
+                open: 'td_open',
+                close: 'td_close',
+            };
+        },
+        table_header(node) {
+            return {
+                open: 'th_open',
+                close: 'th_close',
+            };
+        },
+        br(node) {
+            return {
+                selfClose: 'hardbreak',
+            };
+        },
+        text(node) {
+            return {
+                selfClose: (node) => {
+                    const token = new Token('text', '', 0);
+                    token.content = node.text || '';
+                    return token;
+                },
+            };
+        },
+        code_block(node) {
+            return {
+                selfClose: (node) => {
+                    const token = new Token('code_block', 'code', 0);
+                    token.attrSet('lang', node.attrs.lang);
+                    token.content = '';
+                    node.forEach((child, offset) => {
+                        token.content += child.text || '';
+                    });
+                    return token;
+                },
+            };
+        },
+        // html(state, node) {
+        //   const domSerializer = DOMSerializer.fromSchema(schema);
+        //   const element = domSerializer.serializeNode(node, {
+        //     document: globalThis.document,
+        //   });
+        //   convertDomToLowerCase(element);
+        //   const xmlSerializer = new XMLSerializer();
+        //   const html = xmlSerializer.serializeToString(element) + '\n';
+        //   // state.write(html.replace(/\sxmlns="[^"]*"/, ''));
+        //   state.write(html);
+        // },
+        //
+        image(node) {
+            return {
+                selfClose: (node) => {
+                    const token = new Token('image', 'img', 0);
+                    token.attrSet('src', node.attrs.src);
+                    if (node.attrs.title) {
+                        token.attrSet('title', node.attrs.title);
+                    }
+                    return token;
+                },
+            };
+        },
+    }, {
+        em: {
+            open: 'em_open',
+            close: 'em_close',
+            mixable: true,
+            expelEnclosingWhitespace: true,
+        },
+        underline: {
+            open: 'underline_open',
+            close: 'underline_close',
+            mixable: true,
+            expelEnclosingWhitespace: true,
+        },
+        strong: {
+            open: 'strong_open',
+            close: 'strong_close',
+            mixable: true,
+            expelEnclosingWhitespace: true,
+        },
+        strike: {
+            open: 'strike_open',
+            close: 'strike_close',
+            mixable: true,
+            expelEnclosingWhitespace: true,
+        },
+        math: {
+            open: 'math_inline_open',
+            close: 'math_inline_close',
+            // math_inline
+            // selfClose: (node) => {
+            //   const token = new Token('math', 'math', 0);
+            //   token.attrSet('content', node.attrs.content);
+            //   if (node.attrs.type) {
+            //   }
+            //   return token;
+            // },
+        },
+        link: {
+            open: (mark) => {
+                const token = new Token('link_open', 'a', 1);
+                token.attrSet('href', mark.attrs.href);
+                return token;
+            },
+            close: 'link_close',
+            mixable: true,
+        },
+        code: {
+            // selfClose: (node) => {
+            //   const token = new Token('code_inline', 'code', 0);
+            //   token.content = 'TODO_INLINE_CODE';
+            //   // token.attrSet('content', node.attrs.content);
+            //   // if (node.attrs.type) {
+            //   // }
+            //   return token;
+            // },
+            // open: (mark: Mark) => {
+            // },
+            // close: (mark: Mark) => {
+            //   const token = new Token('code_inline', 'code', 0);
+            //   token.content = 'TODO_INLINE_CODE';
+            //   // token.attrSet('content', node.attrs.content);
+            //   // if (node.attrs.type) {
+            //   // }
+            //   return token;
+            // },
+            open: 'code_open',
+            close: 'code_close',
+            escape: false,
+        },
+        // code: {
+        //   open(_state, _mark, parent, index) {
+        //     return backticksFor(parent.child(index), -1);
+        //   },
+        //   close(_state, _mark, parent, index) {
+        //     return backticksFor(parent.child(index - 1), 1);
+        //   },
+        //   escape: false,
+        // },            // bookmark: {
+        //   open(state, mark) {
+        //     const id = mark.attrs.id;
+        //     if (state.alreadyOutputedIds.has(id)) {
+        //       return '';
+        //     }
+        //     state.alreadyOutputedIds.add(id);
+        //     return `<a id="${id}"></a>`;
+        //   },
+        //   close() {
+        //     return '';
+        //   },
+        // },
+    });
+    document = removeMarkedContent(document, schema.marks.change);
+    // deleteAllMarkedText('change', state, dispatch)
+    const tokens = defaultMarkdownTokenizer.serialize(document);
+    if (config.debugTokens) {
+        const event = new CustomEvent('md:tokens', {
+            detail: {
+                tokens,
+            },
+        });
+        editor.dispatchEvent(event);
+    }
+    const markdownSerializerConfig = {
+        debug: config.serializerDebug,
+    };
+    const serializer = new MarkdownSerializer(markdownSerializerConfig);
+    const output = serializer.serialize(tokens);
+    const event = new CustomEvent('md:output', {
+        detail: {
+            output,
+        },
+    });
+    editor.dispatchEvent(event);
+    const debugMap = {};
+    const rawTextMap = [];
+    let sourceMap;
+    if (config.sourceMap) {
+        // https://sourcemaps.info/spec.html
+        // https://evanw.github.io/source-map-visualization/
+        // https://github.com/evanw/source-map-visualization/blob/gh-pages/code.js
+        const debugOutput = new SmartOutput();
+        nodeToTreeStringOutput(debugOutput, document);
+        debugOutput.getSourceMap((targetRow, targetCol, pos, item) => {
+            if (!item) {
+                return;
+            }
+            debugMap[item.pos] = {
+                targetRow,
+                targetCol,
+            };
+        });
+        sourceMap = output.getSourceMap((targetRow, targetCol, targetPos, item) => {
+            if (item?.map) {
+                const pos = item.map[0];
+                const sourceCol = item.map[2];
+                rawTextMap.push({
+                    nodeIdx: pos,
+                    targetPos,
+                    targetRow,
+                    targetCol,
+                    sourceCol,
+                });
+                if (debugMap[pos]) {
+                    return {
+                        sourceNo: 0,
+                        sourceRowPos: debugMap[pos].targetRow,
+                        sourceColPos: debugMap[pos].targetCol,
+                    };
+                }
+            }
+        });
+        sourceMap.file = 'target.md';
+        sourceMap.sources = ['debug.txt'];
+        sourceMap.sourcesContent = [debugOutput.toString()];
+    }
+    if (config.dispatchSourceMap) {
+        const event = new CustomEvent('md:sourcemap', {
+            detail: {
+                sourceMap,
+                debugMap,
+                rawTextMap,
+            },
+        });
+        editor.dispatchEvent(event);
+    }
+    return {
+        content: output.toString(),
+        sourceMap,
+        debugMap,
+        rawTextMap,
+    };
+}

package/esm/extension-markdown/src/token_handlers/basic_token_handlers.d.ts

@@ -0,0 +1,4 @@
+import type { TokenHandler } from '../MarkdownSerializer.js';
+export declare function getHtmlBasicTokensHandlers(): Record<string, Array<TokenHandler>>;
+export declare function getBasicTokensHandlers(): Record<string, Array<TokenHandler>>;
+//# sourceMappingURL=basic_token_handlers.d.ts.map

package/esm/extension-markdown/src/token_handlers/basic_token_handlers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"basic_token_handlers.d.ts","sourceRoot":"","sources":["../../../../src/extension-markdown/src/token_handlers/basic_token_handlers.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAEV,YAAY,EACb,MAAM,0BAA0B,CAAC;AAGlC,wBAAgB,0BAA0B,IAAI,MAAM,CAClD,MAAM,EACN,KAAK,CAAC,YAAY,CAAC,CACpB,CAuDA;AAgBD,wBAAgB,sBAAsB,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC,CA4G5E"}

package/esm/extension-markdown/src/token_handlers/basic_token_handlers.js

@@ -0,0 +1,151 @@
+import { getInlineTokensHandlers } from './inline_token_handlers.js';
+export function getHtmlBasicTokensHandlers() {
+    return {
+        'frontmatter': [],
+        'heading_open': [
+            (token, ctx) => {
+                ctx.current.log(`<${token.tag}>`, token);
+            },
+        ],
+        'heading_close': [
+            (token, ctx) => {
+                ctx.current.log(`</${token.tag}>\n`, token);
+            },
+        ],
+        'paragraph_open': [
+            (token, ctx) => {
+                ctx.current.log('<p>', token);
+            },
+        ],
+        'paragraph_close': [
+            (token, ctx) => {
+                ctx.current.log('</p>\n', token);
+            },
+        ],
+        'fence': [
+            (token, ctx) => {
+                ctx.current.log('<pre>' + token.content + '</pre>\n', token);
+                // ctx.current.log('```' + token.info + '\n' + token.content + '```\n\n', token);
+            },
+        ],
+        'code_block': [
+            (token, ctx) => {
+                ctx.current.log('<pre>' + token.content + '</pre>\n', token);
+                // ctx.current.log(
+                //   token.content
+                //     .split('\n')
+                //     .map((t) => t ? ('    ' + t) : '')
+                //     .join('\n') + '\n', token
+                // );
+            },
+        ],
+        'hr': [
+            (token, ctx) => {
+                ctx.current.log('<hr />\n', token);
+            },
+        ],
+        'html_block': [
+            (token, ctx) => {
+                ctx.current.log(token.content, token);
+            },
+        ],
+    };
+}
+function getHeaderTokensHandlers() {
+    const entriesBasic = Object.entries(getBasicTokensHandlers())
+        .filter((entry) => ['heading_close'].includes(entry[0]));
+    const entriesInline = Object.entries(getInlineTokensHandlers())
+        .filter((entry) => ['text', 'link_open', 'link_close'].includes(entry[0]));
+    return {
+        ...Object.fromEntries(entriesInline),
+        ...Object.fromEntries(entriesBasic),
+        'default': [],
+    };
+}
+export function getBasicTokensHandlers() {
+    return {
+        'frontmatter': [
+            (token, ctx) => {
+                ctx.current.log('---' + '\n' + token.content + '\n---\n\n', token);
+            },
+        ],
+        'heading_open': [
+            (token, ctx) => {
+                ctx.stash('getBasicTokensHandlers.heading_open');
+                ctx.current.handlers = getHeaderTokensHandlers();
+                if (token.markup === '---') {
+                    return;
+                }
+                ctx.current.log('#'.repeat(+token.tag.substring(1)) + ' ', token);
+            },
+        ],
+        'heading_close': [
+            (token, ctx) => {
+                if (token.markup === '---') {
+                    ctx.current.log('\n' + token.markup + '\n', token);
+                }
+                ctx.current.log('\n', token);
+                ctx.unstash('getBasicTokensHandlers.heading_close');
+            },
+        ],
+        'paragraph_open': [
+            (token, ctx) => {
+                ctx.current.log('', token);
+                // Skip, as paragraphs are implied by newlines
+            },
+        ],
+        'paragraph_close': [
+            (token, ctx) => {
+                if (ctx.output.colPos !== 0) {
+                    ctx.current.log('\n', token);
+                }
+            },
+        ],
+        'fence': [
+            (token, ctx) => {
+                const content = token.content.endsWith('\n')
+                    ? token.content
+                    : token.content + '\n';
+                if (token.info === 'latex') {
+                    ctx.current.log('$$' + '\n' + content + '$$\n\n', token);
+                }
+                else {
+                    const lang = token.attrGet('lang') || '';
+                    ctx.current.log('```' + lang + '\n' + content + '```\n\n', token);
+                }
+            },
+        ],
+        'code_block': [
+            (token, ctx) => {
+                const indent = +(token.attrGet('indent') || 0);
+                const lang = token.attrGet('lang') || '';
+                if (indent === 0) {
+                    const content = token.content.endsWith('\n')
+                        ? token.content
+                        : token.content + '\n';
+                    if (lang === 'latex') {
+                        ctx.current.log('$$' + '\n' + content + '$$\n\n', token);
+                    }
+                    else {
+                        ctx.current.log('```' + lang + '\n' + content + '```\n\n', token);
+                    }
+                    return;
+                }
+                ctx.current.log(token.content
+                    .split('\n')
+                    .map((t) => t ? (' '.repeat(indent) + t) : '')
+                    .join('\n'), token);
+            },
+        ],
+        'hr': [
+            (token, ctx) => {
+                ctx.current.log((token.markup || '---') + '\n', token);
+            },
+        ],
+        'html_block': [
+            (token, ctx) => {
+                ctx.current.log(token.content, token);
+            },
+        ],
+    };
+}

package/esm/extension-markdown/src/token_handlers/footnote_token_handlers.d.ts

@@ -0,0 +1,3 @@
+import type { TokenHandler } from '../MarkdownSerializer.js';
+export declare function getFootnoteTokensHandlers(): Record<string, Array<TokenHandler>>;
+//# sourceMappingURL=footnote_token_handlers.d.ts.map

package/esm/extension-markdown/src/token_handlers/footnote_token_handlers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"footnote_token_handlers.d.ts","sourceRoot":"","sources":["../../../../src/extension-markdown/src/token_handlers/footnote_token_handlers.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAEV,YAAY,EACb,MAAM,0BAA0B,CAAC;AAElC,wBAAgB,yBAAyB,IAAI,MAAM,CACjD,MAAM,EACN,KAAK,CAAC,YAAY,CAAC,CACpB,CAiCA"}

package/esm/extension-markdown/src/token_handlers/footnote_token_handlers.js

@@ -0,0 +1,34 @@
+export function getFootnoteTokensHandlers() {
+    return {
+        'footnote_block_open': [
+            (token, ctx) => {
+            },
+        ],
+        'footnote_block_close': [
+            (token, ctx) => {
+            },
+        ],
+        'footnote_open': [
+            (token, ctx) => {
+                if (token.meta.label) {
+                    ctx.current.log(`[^${token.meta.label}]: `, token);
+                }
+                else {
+                    ctx.current.log(`[^footnote_${token.meta.id}]: `, token);
+                }
+                ctx.stash('getFootnoteTokensHandlers.footnote_open');
+                ctx.current.footnoteCnt++;
+                ctx.current.itemRow = 0;
+            },
+        ],
+        'footnote_close': [
+            (token, ctx) => {
+                ctx.unstash('getFootnoteTokensHandlers.footnote_close');
+            },
+        ],
+        'footnote_anchor': [
+            (token, ctx) => {
+            },
+        ],
+    };
+}

package/esm/extension-markdown/src/token_handlers/inline_token_handlers.d.ts

@@ -0,0 +1,6 @@
+import type { Token } from '../types.js';
+import type { TokenHandler } from '../MarkdownSerializer.js';
+export declare function escapeMarkdown(token: Token): Array<[string, Token]>;
+export declare function getInlineTokensHandlers(): Record<string, Array<TokenHandler>>;
+export declare function getHtmlInlineTokensHandlers(): Record<string, Array<TokenHandler>>;
+//# sourceMappingURL=inline_token_handlers.d.ts.map

package/esm/extension-markdown/src/token_handlers/inline_token_handlers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inline_token_handlers.d.ts","sourceRoot":"","sources":["../../../../src/extension-markdown/src/token_handlers/inline_token_handlers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,KAAK,EAEV,YAAY,EACb,MAAM,0BAA0B,CAAC;AAElC,wBAAgB,cAAc,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CA4EnE;AAwDD,wBAAgB,uBAAuB,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC,CAqJ7E;AAMD,wBAAgB,2BAA2B,IAAI,MAAM,CACnD,MAAM,EACN,KAAK,CAAC,YAAY,CAAC,CACpB,CA4IA"}