@kerebron/extension-codemirror 0.3.1 → 0.4.1

package/esm/extension-lsp/src/workspace.d.ts

@@ -0,0 +1,62 @@
+import type * as lsp from 'vscode-languageserver-protocol';
+import type { EditorUi } from '../../editor/src/mod.js';
+import { LSPClient } from './LSPClient.js';
+import { PositionMapper } from '../../extension-markdown/src/PositionMapper.js';
+export interface LspSource {
+    ui: EditorUi;
+    getMappedContent(): {
+        content: string;
+        mapper: PositionMapper;
+    };
+}
+export interface WorkspaceFile {
+    uri: string;
+    languageId: string;
+    version: number;
+    content: string;
+    source: LspSource;
+    getUi(): EditorUi | undefined;
+    mapper: PositionMapper;
+}
+interface WorkspaceFileUpdate {
+    file: WorkspaceFile;
+    prevDoc: string;
+    changes: lsp.TextDocumentContentChangeEvent[];
+}
+export declare abstract class Workspace {
+    readonly client: LSPClient;
+    abstract files: WorkspaceFile[];
+    protected isConnected: boolean;
+    constructor(client: LSPClient);
+    getFile(uri: string): WorkspaceFile | null;
+    abstract syncFiles(): readonly WorkspaceFileUpdate[];
+    requestFile(uri: string): Promise<WorkspaceFile | null>;
+    abstract openFile(uri: string, languageId: string, source: LspSource): void;
+    abstract changedFile(uri: string): void;
+    abstract closeFile(uri: string): void;
+    connected(): Promise<void>;
+    disconnected(): void;
+    getUi(uri: string): Promise<EditorUi | undefined>;
+}
+declare class DefaultWorkspaceFile implements WorkspaceFile {
+    readonly uri: string;
+    readonly languageId: string;
+    version: number;
+    content: string;
+    mapper: PositionMapper;
+    source: LspSource;
+    syncedContent: string;
+    constructor(uri: string, languageId: string, version: number, content: string, mapper: PositionMapper, source: LspSource);
+    getUi(): EditorUi;
+}
+export declare class DefaultWorkspace extends Workspace {
+    files: DefaultWorkspaceFile[];
+    private fileVersions;
+    nextFileVersion(uri: string): number;
+    syncFiles(): WorkspaceFileUpdate[];
+    changedFile(uri: string): Promise<void>;
+    openFile(uri: string, languageId: string, source: LspSource): void;
+    closeFile(uri: string): void;
+}
+export {};
+//# sourceMappingURL=workspace.d.ts.map

package/esm/extension-lsp/src/workspace.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"workspace.d.ts","sourceRoot":"","sources":["../../../src/extension-lsp/src/workspace.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,GAAG,MAAM,gCAAgC,CAAC;AAG3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAExD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,gDAAgD,CAAC;AAGhF,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,QAAQ,CAAC;IACb,gBAAgB,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,cAAc,CAAA;KAAE,CAAC;CACjE;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,KAAK,IAAI,QAAQ,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,cAAc,CAAC;CACxB;AAED,UAAU,mBAAmB;IAC3B,IAAI,EAAE,aAAa,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,GAAG,CAAC,8BAA8B,EAAE,CAAC;CAC/C;AAED,8BAAsB,SAAS;IAK3B,QAAQ,CAAC,MAAM,EAAE,SAAS;IAJ5B,QAAQ,CAAC,KAAK,EAAE,aAAa,EAAE,CAAC;IAChC,SAAS,CAAC,WAAW,UAAS;gBAGnB,MAAM,EAAE,SAAS;IAG5B,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAI1C,QAAQ,CAAC,SAAS,IAAI,SAAS,mBAAmB,EAAE;IAEpD,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAIvD,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,IAAI;IAC3E,QAAQ,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IACvC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAE/B,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAOhC,YAAY,IAAI,IAAI;IAOpB,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;CAIlD;AAED,cAAM,oBAAqB,YAAW,aAAa;IAG/C,QAAQ,CAAC,GAAG,EAAE,MAAM;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM;IACpB,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,cAAc;IACtB,MAAM,EAAE,SAAS;IAPnB,aAAa,EAAE,MAAM,CAAM;gBAEvB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,SAAS;IAI1B,KAAK;CAGN;AAED,qBAAa,gBAAiB,SAAQ,SAAS;IAC7C,KAAK,EAAE,oBAAoB,EAAE,CAAM;IACnC,OAAO,CAAC,YAAY,CAAkD;IAEtE,eAAe,CAAC,GAAG,EAAE,MAAM;IAI3B,SAAS;IAaH,WAAW,CAAC,GAAG,EAAE,MAAM;IAkC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS;IAsB3D,SAAS,CAAC,GAAG,EAAE,MAAM;CAOtB"}

package/esm/extension-lsp/src/workspace.js

@@ -0,0 +1,168 @@
+import { TextDocumentSyncKind } from 'vscode-languageserver-protocol';
+import { computeIncrementalChanges } from './computeIncrementalChanges.js';
+export class Workspace {
+    constructor(client) {
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: client
+        });
+        Object.defineProperty(this, "isConnected", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+    }
+    getFile(uri) {
+        return this.files.find((f) => f.uri == uri) || null;
+    }
+    requestFile(uri) {
+        return Promise.resolve(this.getFile(uri));
+    }
+    async connected() {
+        this.isConnected = true;
+        for await (const file of this.files) {
+            const result = await this.client.didOpen(file);
+        }
+    }
+    disconnected() {
+        for (const file of this.files) {
+            this.client.workspace.closeFile(file.uri);
+        }
+        this.isConnected = false;
+    }
+    getUi(uri) {
+        const file = this.getFile(uri);
+        return Promise.resolve(file ? file.getUi() : undefined);
+    }
+}
+class DefaultWorkspaceFile {
+    constructor(uri, languageId, version, content, mapper, source) {
+        Object.defineProperty(this, "uri", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: uri
+        });
+        Object.defineProperty(this, "languageId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: languageId
+        });
+        Object.defineProperty(this, "version", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: version
+        });
+        Object.defineProperty(this, "content", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: content
+        });
+        Object.defineProperty(this, "mapper", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: mapper
+        });
+        Object.defineProperty(this, "source", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: source
+        });
+        Object.defineProperty(this, "syncedContent", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ''
+        });
+    }
+    getUi() {
+        return this.source.ui;
+    }
+}
+export class DefaultWorkspace extends Workspace {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "files", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "fileVersions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: Object.create(null)
+        });
+    }
+    nextFileVersion(uri) {
+        return this.fileVersions[uri] = (this.fileVersions[uri] ?? -1) + 1;
+    }
+    syncFiles() {
+        if (!this.client.supportSync) {
+            return [];
+        }
+        const result = [];
+        for (const file of this.files) {
+            this.changedFile(file.uri);
+        }
+        return result;
+    }
+    async changedFile(uri) {
+        const file = this.files.find((f) => f.uri == uri) || null;
+        if (file) {
+            const { content, mapper } = file.source.getMappedContent();
+            if (!this.isConnected) {
+                file.content = content;
+                file.mapper = mapper;
+                return;
+            }
+            if (await this.client.notification('textDocument/didChange', {
+                textDocument: { uri: file.uri, version: file.version },
+                contentChanges: contentChangesFor(file, content, mapper, this.client.supportSync == TextDocumentSyncKind.Incremental),
+            })) {
+                file.syncedContent = file.content;
+                file.content = content;
+                file.mapper = mapper;
+                file.version = this.nextFileVersion(file.uri);
+            }
+        }
+    }
+    openFile(uri, languageId, source) {
+        // if (uri) {}
+        if (this.getFile(uri)) {
+            this.closeFile(uri);
+        }
+        const mappedContent = source.getMappedContent();
+        const { content, mapper } = mappedContent;
+        const file = new DefaultWorkspaceFile(uri, languageId, this.nextFileVersion(uri), content, mapper, source);
+        this.files.push(file);
+        this.client.didOpen(file);
+    }
+    closeFile(uri) {
+        const file = this.getFile(uri);
+        if (file) {
+            this.files = this.files.filter((f) => f.uri !== uri);
+            this.client.didClose(uri);
+        }
+    }
+}
+var Sync;
+(function (Sync) {
+    Sync[Sync["AlwaysIfSmaller"] = 1024] = "AlwaysIfSmaller";
+})(Sync || (Sync = {}));
+function contentChangesFor(file, newContent, mapper, supportInc) {
+    if (!supportInc || newContent.length < Sync.AlwaysIfSmaller) {
+        return [{ text: newContent }];
+    }
+    const changes = computeIncrementalChanges(file.content, newContent);
+    return changes.reverse();
+}

package/esm/extension-markdown/src/DocumentMarkdownInlineTokenizer.d.ts

@@ -0,0 +1,36 @@
+import type { Mark, Node } from 'prosemirror-model';
+import { Token } from './types.js';
+export type MarkTokenizerSpec = {
+    open?: string | ((mark: Mark) => Token);
+    close?: string | ((mark: Mark) => Token);
+    mixable?: boolean;
+    expelEnclosingWhitespace?: boolean;
+    escape?: boolean;
+};
+export type DocumentMarkdownTokenizerSpec = {
+    open?: string | ((node: Node) => Token);
+    close?: string | ((node: Node) => Token);
+    selfClose?: string | ((node: Node) => Token);
+};
+export declare class DocumentMarkdownInlineTokenizer {
+    readonly nodes: {
+        [node: string]: (node: Node, index: number) => DocumentMarkdownTokenizerSpec;
+    };
+    readonly marks: {
+        [mark: string]: MarkTokenizerSpec;
+    };
+    readonly options: {
+        hardBreakNodeName: string;
+    };
+    constructor(nodes: {
+        [node: string]: (node: Node, index: number) => DocumentMarkdownTokenizerSpec;
+    }, marks: {
+        [mark: string]: MarkTokenizerSpec;
+    }, options: {
+        hardBreakNodeName: string;
+    });
+    private getMark;
+    renderInline(parent: Node, currentPos: number, level: number): Array<Token>;
+    markString(mark: Mark, open: boolean, tokens: Token[]): void;
+}
+//# sourceMappingURL=DocumentMarkdownInlineTokenizer.d.ts.map

package/esm/extension-markdown/src/DocumentMarkdownInlineTokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DocumentMarkdownInlineTokenizer.d.ts","sourceRoot":"","sources":["../../../src/extension-markdown/src/DocumentMarkdownInlineTokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,KAAK,CAAC,CAAC;IACxC,KAAK,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,KAAK,CAAC,CAAC;IACzC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,6BAA6B,GAAG;IAC1C,IAAI,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,KAAK,CAAC,CAAC;IACxC,KAAK,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,KAAK,CAAC,CAAC;IACzC,SAAS,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,KAAK,CAAC,CAAC;CAC9C,CAAC;AAuCF,qBAAa,+BAA+B;IAExC,QAAQ,CAAC,KAAK,EAAE;QACd,CAAC,IAAI,EAAE,MAAM,GAAG,CACd,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,KACV,6BAA6B,CAAC;KACpC;IACD,QAAQ,CAAC,KAAK,EAAE;QAAE,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAAA;KAAE;IACrD,QAAQ,CAAC,OAAO,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE;gBAPtC,KAAK,EAAE;QACd,CAAC,IAAI,EAAE,MAAM,GAAG,CACd,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,KACV,6BAA6B,CAAC;KACpC,EACQ,KAAK,EAAE;QAAE,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAAA;KAAE,EAC5C,OAAO,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE;IAIjD,OAAO,CAAC,OAAO;IAaf,YAAY,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;IAwK3E,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE;CAoCtD"}

package/esm/extension-markdown/src/DocumentMarkdownInlineTokenizer.js

@@ -0,0 +1,240 @@
+import { Token } from './types.js';
+const blankMark = { open: '', close: '', mixable: true };
+const blankNode = { open: '', close: '' };
+function generateInlineTokens(input) {
+    const tokens = [];
+    let currentPos = -1;
+    let currentText = '';
+    for (const [text, inlinePos] of input) {
+        if (true || currentPos !== inlinePos) {
+            if (currentText.length > 0) {
+                const token = new Token('text', '', 0);
+                token.meta = 'generateInline1';
+                token.map = [currentPos];
+                token.content = currentText;
+                tokens.push(token);
+            }
+            currentText = '';
+            currentPos = inlinePos;
+        }
+        currentText += text;
+    }
+    if (currentText.length > 0) {
+        const token = new Token('text', '', 0);
+        token.meta = 'generateInline2';
+        token.map = [currentPos];
+        token.content = currentText;
+        tokens.push(token);
+    }
+    return tokens;
+}
+export class DocumentMarkdownInlineTokenizer {
+    constructor(nodes, marks, options) {
+        Object.defineProperty(this, "nodes", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: nodes
+        });
+        Object.defineProperty(this, "marks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: marks
+        });
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: options
+        });
+    }
+    getMark(name) {
+        let info = this.marks[name];
+        if (!info) {
+            // if (this.options.strict !== false) {
+            //   throw new Error(
+            //     `Mark type \`${name}\` not supported by Markdown renderer`,
+            //   );
+            // }
+            return blankMark;
+        }
+        return info;
+    }
+    renderInline(parent, currentPos, level) {
+        const active = [];
+        const trailing = [];
+        const inlineTokens = [];
+        const progress = (node, offset, index) => {
+            const inlinePos = currentPos + offset;
+            let marks = node ? node.marks : [];
+            // Remove marks from `hard_break` that are the last node inside
+            // that mark to prevent parser edge cases with new lines just
+            // before closing marks.
+            // TODO:
+            if (node && node.type.name === this.options.hardBreakNodeName) {
+                marks = marks.filter((m) => {
+                    if (index + 1 == parent.childCount)
+                        return false;
+                    let next = parent.child(index + 1);
+                    return m.isInSet(next.marks) &&
+                        (!next.isText || /\S/.test(next.text));
+                });
+            }
+            const leading = trailing.splice(0, trailing.length);
+            // If whitespace has to be expelled from the node, adjust
+            // leading and trailing accordingly.
+            if (node.isText && marks.some((mark) => {
+                let info = this.getMark(mark.type.name);
+                return info && info.expelEnclosingWhitespace && !mark.isInSet(active);
+            })) {
+                let [_, lead, rest] = /^(\s*)(.*)$/m.exec(node.text);
+                if (lead) {
+                    leading.push([lead, inlinePos]);
+                    node = rest ? node.withText(rest) : null;
+                    if (!node)
+                        marks = active;
+                }
+            }
+            if (node.isText && marks.some((mark) => {
+                let info = this.getMark(mark.type.name);
+                return info && info.expelEnclosingWhitespace &&
+                    (index == parent.childCount - 1 ||
+                        !mark.isInSet(parent.child(index + 1).marks));
+            })) {
+                let [_, rest, trail] = /^(.*?)(\s*)$/m.exec(node.text);
+                if (trail) {
+                    trailing.push([trail, inlinePos]);
+                    node = rest ? node.withText(rest) : null;
+                    if (!node)
+                        marks = active;
+                }
+            }
+            let inner = marks.length ? marks[marks.length - 1] : null;
+            let noEsc = inner && this.getMark(inner.type.name).escape === false;
+            let len = marks.length - (noEsc ? 1 : 0);
+            // Try to reorder 'mixable' marks, such as em and strong, which
+            // in Markdown may be opened and closed in different order, so
+            // that order of the marks for the token matches the order in
+            // active.
+            outer: for (let i = 0; i < len; i++) {
+                let mark = marks[i];
+                if (!this.getMark(mark.type.name).mixable)
+                    break;
+                for (let j = 0; j < active.length; j++) {
+                    let other = active[j];
+                    if (!this.getMark(other.type.name).mixable)
+                        break;
+                    if (mark.eq(other)) {
+                        if (i > j) {
+                            marks = marks.slice(0, j).concat(mark).concat(marks.slice(j, i))
+                                .concat(marks.slice(i + 1, len));
+                        }
+                        else if (j > i) {
+                            marks = marks.slice(0, i).concat(marks.slice(i + 1, j)).concat(mark).concat(marks.slice(j, len));
+                        }
+                        continue outer;
+                    }
+                }
+            }
+            // Find the prefix of the mark set that didn't change
+            let keep = 0;
+            while (keep < Math.min(active.length, len) && marks[keep].eq(active[keep]))
+                ++keep;
+            // Close the marks that need to be closed
+            while (keep < active.length) {
+                const x = active.pop();
+                this.markString(x, false, inlineTokens);
+            }
+            // Output any previously expelled trailing whitespace outside the marks
+            inlineTokens.push(...generateInlineTokens(leading.splice(0, leading.length)));
+            // Open the marks that need to be opened
+            while (active.length < len) {
+                let add = marks[active.length];
+                active.push(add);
+                this.markString(add, true, inlineTokens);
+            }
+            // Render the node. Special case code marks, since their content
+            // may not be escaped.
+            if (noEsc && node.isText) {
+                this.markString(inner, true, inlineTokens);
+                if (node.text) {
+                    const token = new Token('text', '', 0);
+                    token.meta = 'noEscText';
+                    token.map = [inlinePos];
+                    token.content = node.text;
+                    inlineTokens.push(token);
+                }
+                this.markString(inner, false, inlineTokens);
+            }
+            else {
+                const nodeSpec = this.nodes[node.type.name]
+                    ? this.nodes[node.type.name](node, currentPos)
+                    : blankNode;
+                let tag = '';
+                if (node.type.spec.toDOM) {
+                    const dom = node.type.spec.toDOM(node);
+                    if (Array.isArray(dom) && dom.length > 0) {
+                        tag = dom[0];
+                    }
+                }
+                if (nodeSpec.selfClose) {
+                    if (typeof nodeSpec.selfClose === 'string') {
+                        const token = new Token(nodeSpec.selfClose, tag, 0);
+                        token.meta = 'nodeSpec.selfClose';
+                        token.level = level;
+                        token.map = [inlinePos];
+                        inlineTokens.push(token);
+                    }
+                    else {
+                        const token = nodeSpec.selfClose(node);
+                        token.meta = '!nodeSpec.selfClose';
+                        token.level = level;
+                        token.map = [inlinePos];
+                        inlineTokens.push(token);
+                    }
+                }
+            }
+        };
+        parent.forEach((child, offset, index) => progress(child, offset, index));
+        // Close the marks that need to be closed
+        while (active.length > 0) {
+            const x = active.pop();
+            this.markString(x, false, inlineTokens);
+        }
+        // Output any previously expelled trailing whitespace outside the marks
+        inlineTokens.push(...generateInlineTokens(trailing.splice(0, trailing.length)));
+        return inlineTokens;
+    }
+    markString(mark, open, tokens) {
+        let info = this.getMark(mark.type.name);
+        let value = open ? info.open : info.close;
+        if (!value) {
+            /** Skip unknown marks (like textColor, highlight) - they'll be lost in markdown but won't crash */
+            console.warn('Unsupported mark type for markdown: ' + mark.type.name +
+                ', available types: ' +
+                Object.keys(this.marks).join(', ') + '. Mark will be ignored.');
+            return;
+        }
+        const token = typeof value == 'string' ? value : value(mark);
+        let tag = '';
+        if (mark.type.spec.toDOM) {
+            const dom = mark.type.spec.toDOM(mark, true);
+            if (Array.isArray(dom) && dom.length > 0) {
+                tag = dom[0];
+            }
+        }
+        if (typeof value === 'string') {
+            const token = new Token(value, tag, open ? 1 : -1);
+            token.meta = 'markString';
+            tokens.push(token);
+        }
+        else {
+            const token = value(mark);
+            if (token) {
+                token.meta = token.meta || 'markString()';
+                tokens.push(token);
+            }
+        }
+    }
+}

package/esm/extension-markdown/src/DocumentMarkdownTokenizer.d.ts

@@ -0,0 +1,26 @@
+import type { Node } from 'prosemirror-model';
+import { Token } from './types.js';
+import { DocumentMarkdownInlineTokenizer, DocumentMarkdownTokenizerSpec, MarkTokenizerSpec } from './DocumentMarkdownInlineTokenizer.js';
+export declare class DocumentMarkdownTokenizer {
+    readonly nodes: {
+        [node: string]: (node: Node, index: number) => DocumentMarkdownTokenizerSpec;
+    };
+    readonly marks: {
+        [mark: string]: MarkTokenizerSpec;
+    };
+    readonly options: {
+        hardBreakNodeName: string;
+    };
+    private tokens;
+    inlineTokenizer: DocumentMarkdownInlineTokenizer;
+    constructor(nodes: {
+        [node: string]: (node: Node, index: number) => DocumentMarkdownTokenizerSpec;
+    }, marks: {
+        [mark: string]: MarkTokenizerSpec;
+    }, options?: {
+        hardBreakNodeName: string;
+    });
+    iterateNode(node: Node, currentPos: number, level?: number): void;
+    serialize(content: Node, options?: {}): Token[];
+}
+//# sourceMappingURL=DocumentMarkdownTokenizer.d.ts.map

package/esm/extension-markdown/src/DocumentMarkdownTokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DocumentMarkdownTokenizer.d.ts","sourceRoot":"","sources":["../../../src/extension-markdown/src/DocumentMarkdownTokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EACL,+BAA+B,EAC/B,6BAA6B,EAC7B,iBAAiB,EAClB,MAAM,sCAAsC,CAAC;AAI9C,qBAAa,yBAAyB;IAKlC,QAAQ,CAAC,KAAK,EAAE;QACd,CAAC,IAAI,EAAE,MAAM,GAAG,CACd,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,KACV,6BAA6B,CAAC;KACpC;IACD,QAAQ,CAAC,KAAK,EAAE;QAAE,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAAA;KAAE;IACrD,QAAQ,CAAC,OAAO,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE;IAXjD,OAAO,CAAC,MAAM,CAAoB;IAClC,eAAe,EAAE,+BAA+B,CAAC;gBAGtC,KAAK,EAAE;QACd,CAAC,IAAI,EAAE,MAAM,GAAG,CACd,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,KACV,6BAA6B,CAAC;KACpC,EACQ,KAAK,EAAE;QAAE,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAAA;KAAE,EAC5C,OAAO,GAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAE5C;IASH,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,SAAK;IA8EtD,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,GAAE,EAAO,GAAG,KAAK,EAAE;CAIpD"}

package/esm/extension-markdown/src/DocumentMarkdownTokenizer.js

@@ -0,0 +1,115 @@
+import { Token } from './types.js';
+import { DocumentMarkdownInlineTokenizer, } from './DocumentMarkdownInlineTokenizer.js';
+const blankNode = { open: '', close: '' };
+export class DocumentMarkdownTokenizer {
+    constructor(nodes, marks, options = {
+        hardBreakNodeName: 'hard_break',
+    }) {
+        Object.defineProperty(this, "nodes", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: nodes
+        });
+        Object.defineProperty(this, "marks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: marks
+        });
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: options
+        });
+        Object.defineProperty(this, "tokens", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "inlineTokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.inlineTokenizer = new DocumentMarkdownInlineTokenizer(nodes, marks, options);
+    }
+    iterateNode(node, currentPos, level = -1) {
+        const nodeSpec = this.nodes[node.type.name]
+            ? this.nodes[node.type.name](node, currentPos)
+            : blankNode;
+        let tag = '';
+        if (node.type.spec.toDOM) {
+            const dom = node.type.spec.toDOM(node);
+            if (Array.isArray(dom) && dom.length > 0) {
+                tag = dom[0];
+            }
+        }
+        if (nodeSpec.open) {
+            if (typeof nodeSpec.open === 'string') {
+                const token = new Token(nodeSpec.open, tag, 1);
+                token.level = level;
+                token.meta = 'nodeSpec.open';
+                token.map = [currentPos];
+                this.tokens.push(token);
+            }
+            else {
+                const token = nodeSpec.open(node);
+                token.level = level;
+                token.meta = 'nodeSpec.open()';
+                token.map = [currentPos];
+                this.tokens.push(token);
+            }
+        }
+        if (nodeSpec.selfClose) {
+            if (typeof nodeSpec.selfClose === 'string') {
+                const token = new Token(nodeSpec.selfClose, tag, 0);
+                token.level = level;
+                token.map = [currentPos];
+                this.tokens.push(token);
+            }
+            else {
+                const token = nodeSpec.selfClose(node);
+                token.level = level;
+                token.map = [currentPos];
+                this.tokens.push(token);
+            }
+            return;
+        }
+        if (node.inlineContent) {
+            const token = new Token('inline', '', 0);
+            token.level = level + 1;
+            token.map = [currentPos];
+            this.tokens.push(token);
+            token.children = this.inlineTokenizer.renderInline(node, currentPos + 1, level + 1);
+        }
+        else if (node.childCount > 0) {
+            node.forEach((child, offset) => {
+                this.iterateNode(child, currentPos + offset + 1, level + 1);
+            });
+        }
+        if (nodeSpec.close) {
+            if (typeof nodeSpec.close === 'string') {
+                const token = new Token(nodeSpec.close, tag, -1);
+                token.meta = 'nodeSpec.close';
+                token.level = level;
+                // token.map = [currentPos];
+                this.tokens.push(token);
+            }
+            else {
+                const token = nodeSpec.close(node);
+                token.meta = 'nodeSpec.close()';
+                token.level = level;
+                // token.map = [currentPos];
+                this.tokens.push(token);
+            }
+        }
+    }
+    serialize(content, options = {}) {
+        this.iterateNode(content, -1); // doc does not have index, so I put -1 for correct calculation
+        return this.tokens;
+    }
+}