@markuplint/parser-utils 4.0.0-dev.28 → 4.0.0-rc.0

package/lib/script-parser.d.ts

@@ -0,0 +1,5 @@
+export declare function scriptParser(script: string): ScriptTokenType[];
+export type ScriptTokenType = {
+    type: 'Identifier' | 'Boolean' | 'Numeric' | 'String' | 'Template' | 'Punctuator';
+    value: string;
+};

package/lib/script-parser.js

@@ -0,0 +1,12 @@
+// @ts-ignore
+import { tokenize } from 'espree';
+export function scriptParser(script) {
+    const tokens = tokenize(script, {
+        ecmaVersion: 'latest',
+        loc: false,
+    });
+    return tokens.map((token) => ({
+        type: token.type,
+        value: token.value,
+    }));
+}

package/lib/sort-nodes.d.ts

@@ -0,0 +1,2 @@
+import type { MLASTNodeTreeItem } from '@markuplint/ml-ast';
+export declare function sortNodes(a: MLASTNodeTreeItem, b: MLASTNodeTreeItem): 1 | -1 | 0;

package/lib/sort-nodes.js

@@ -0,0 +1,18 @@
+export function sortNodes(a, b) {
+    if (a.startOffset === b.startOffset) {
+        return sort(a, b, 'endOffset');
+    }
+    return sort(a, b, 'startOffset');
+}
+function sort(a, b, key) {
+    if (Number.isNaN(a[key]) || Number.isNaN(b[key])) {
+        return 0;
+    }
+    if (a[key] < b[key]) {
+        return -1;
+    }
+    if (a[key] > b[key]) {
+        return 1;
+    }
+    return 0;
+}

package/lib/types.d.ts

@@ -1,14 +1,48 @@
+import type { EndTagType, MLASTParentNode, ParserOptions as ConfigParserOptions } from '@markuplint/ml-ast';
+export type ParserOptions = {
+    readonly booleanish?: boolean;
+    readonly endTagType?: EndTagType;
+    readonly ignoreTags?: readonly IgnoreTag[];
+    readonly maskChar?: string;
+    readonly tagNameCaseSensitive?: boolean;
+    readonly selfCloseType?: SelfCloseType;
+    readonly spaceChars?: readonly string[];
+    readonly rawTextElements?: readonly string[];
+};
+export type ParseOptions = ConfigParserOptions & {
+    readonly offsetOffset?: number;
+    readonly offsetLine?: number;
+    readonly offsetColumn?: number;
+    readonly depth?: number;
+};
+export type Tokenized<N extends {} = {}, State extends unknown = null> = {
+    readonly ast: N[];
+    readonly isFragment: boolean;
+    readonly state?: State;
+};
+export type Token = {
+    readonly raw: string;
+    readonly startOffset: number;
+    readonly startLine: number;
+    readonly startCol: number;
+};
+export type ChildToken = Token & {
+    readonly depth: number;
+    readonly parentNode: MLASTParentNode | null;
+};
+export type SelfCloseType = 'html' | 'xml' | 'html+xml';
 export type Code = {
     readonly type: string;
     readonly index: number;
     readonly startTag: string;
     readonly taggedCode: string;
     readonly endTag: string | null;
+    resolved: boolean;
 };
 export type IgnoreTag = {
     readonly type: string;
-    readonly start: Readonly<RegExp>;
-    readonly end: Readonly<RegExp>;
+    readonly start: Readonly<RegExp> | string;
+    readonly end: Readonly<RegExp> | string;
 };
 export type IgnoreBlock = {
     readonly source: string;
@@ -16,3 +50,7 @@ export type IgnoreBlock = {
     readonly stack: readonly Code[];
     readonly maskChar: string;
 };
+export type QuoteSet = {
+    readonly start: string;
+    readonly end: string;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@markuplint/parser-utils",
-	"version": "4.0.0-dev.28+0131de5e",
+	"version": "4.0.0-rc.0",
 	"description": "Utility module for markuplint parser plugin",
 	"repository": "git@github.com:markuplint/markuplint.git",
 	"author": "Yusuke Hirao <yusukehirao@me.com>",
@@ -10,6 +10,9 @@
 	"exports": {
 		".": {
 			"import": "./lib/index.js"
+		},
+		"./location": {
+			"import": "./lib/get-location.js"
 		}
 	},
 	"types": "lib/index.d.ts",
@@ -24,11 +27,14 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@markuplint/ml-ast": "4.0.0-dev.28+0131de5e",
-		"@markuplint/types": "4.0.0-dev.28+0131de5e",
-		"@types/uuid": "^9.0.6",
-		"type-fest": "^4.5.0",
+		"@markuplint/ml-ast": "4.0.0-rc.0",
+		"@markuplint/ml-spec": "4.0.0-rc.0",
+		"@markuplint/types": "4.0.0-rc.0",
+		"@types/uuid": "^9.0.8",
+		"debug": "^4.3.4",
+		"espree": "^10.0.0",
+		"type-fest": "^4.10.2",
 		"uuid": "^9.0.1"
 	},
-	"gitHead": "0131de5ea9dd6d3fd5472d7b414b66644c758881"
+	"gitHead": "3fdeb45cb69ed52b3a215a7520cea1181601443f"
 }

package/lib/create-token.d.ts

@@ -1,4 +0,0 @@
-import type { MLToken } from '@markuplint/ml-ast';
-export declare function tokenizer(raw: string | null, startLine: number, startCol: number, startOffset: number): MLToken;
-export declare function createTokenFromRawCode(raw: string | null, startOffset: number, rawCode: string): MLToken;
-export declare function uuid(): string;

package/lib/create-token.js

@@ -1,29 +0,0 @@
-import { v4 as uuid4 } from 'uuid';
-import { getEndCol, getEndLine, sliceFragment } from './get-location.js';
-export function tokenizer(raw, startLine, startCol, startOffset) {
-    raw = raw ?? '';
-    const endLine = getEndLine(raw, startLine);
-    const endCol = getEndCol(raw, startCol);
-    const endOffset = startOffset + raw.length;
-    return {
-        uuid: uuid(),
-        raw,
-        startOffset,
-        endOffset,
-        startLine,
-        endLine,
-        startCol,
-        endCol,
-    };
-}
-export function createTokenFromRawCode(raw, startOffset, rawCode) {
-    raw = raw ?? '';
-    const loc = sliceFragment(rawCode, startOffset, startOffset + raw.length);
-    return {
-        uuid: uuid(),
-        ...loc,
-    };
-}
-export function uuid() {
-    return uuid4();
-}

package/lib/flatten-nodes.d.ts

@@ -1,2 +0,0 @@
-import type { MLASTNode } from '@markuplint/ml-ast';
-export declare function flattenNodes(nodeTree: MLASTNode[], rawHtml: string, createLastText?: boolean): MLASTNode[];

package/lib/flatten-nodes.js

@@ -1,247 +0,0 @@
-import { uuid } from './create-token.js';
-import { getEndCol, getEndLine } from './get-location.js';
-import { removeDeprecatedNode } from './remove-deprecated-node.js';
-import tagSplitter from './tag-splitter.js';
-import { walk } from './walker.js';
-export function flattenNodes(
-// eslint-disable-next-line @typescript-eslint/prefer-readonly-parameter-types
-nodeTree, rawHtml, createLastText = true) {
-    const nodeOrders = arrayize(nodeTree, rawHtml);
-    {
-        /**
-         * Correction prev/next/parent
-         */
-        let prevToken = null;
-        for (const node of nodeOrders) {
-            if (!prevToken) {
-                prevToken = node;
-                continue;
-            }
-            if (node.type !== 'endtag') {
-                prevToken = node;
-                continue;
-            }
-            const endTag = node;
-            if (endTag.nodeName.toLowerCase() === 'body' && prevToken.type === 'text') {
-                const prevWreckagesText = prevToken;
-                const wreckages = tagSplitter(prevWreckagesText.raw, prevWreckagesText.startLine, prevWreckagesText.startCol);
-                if (wreckages.length > 0 && wreckages[0]) {
-                    // console.log('wreckages\n', wreckages);
-                    const lastText = wreckages[0];
-                    const raw = lastText.raw;
-                    const startLine = lastText.line;
-                    const startCol = lastText.col;
-                    prevWreckagesText.raw = raw;
-                    prevWreckagesText.endOffset = prevWreckagesText.startOffset + raw.length;
-                    prevWreckagesText.startLine = startLine;
-                    prevWreckagesText.endLine = getEndLine(raw, startLine);
-                    prevWreckagesText.startCol = startCol;
-                    prevWreckagesText.endCol = getEndCol(raw, startCol);
-                }
-            }
-        }
-    }
-    removeDeprecatedNode(nodeOrders);
-    {
-        /**
-         * getting last node
-         */
-        let lastNode = null;
-        for (const node of nodeOrders) {
-            if (node.isGhost) {
-                continue;
-            }
-            lastNode = node;
-        }
-        if (lastNode) {
-            if (lastNode.type === 'text') {
-                // Correction for Parse5 AST
-                // prev node: ? -> html
-                lastNode.prevNode = lastNode.parentNode?.parentNode ?? lastNode.parentNode;
-                if (lastNode.prevNode) {
-                    lastNode.prevNode.nextNode = lastNode;
-                }
-                // parent node: body -> null
-                lastNode.parentNode = null;
-                // next node: ? -> null
-                lastNode.nextNode = null;
-            }
-            else if (createLastText) {
-                /**
-                 * create Last spaces
-                 */
-                let lastOffset = 0;
-                for (const node of nodeOrders) {
-                    lastOffset = Math.max(node.endOffset, lastOffset);
-                }
-                // console.log(lastOffset);
-                const lastTextContent = rawHtml.slice(lastOffset);
-                // console.log(`"${lastTextContent}"`);
-                if (lastTextContent) {
-                    const line = lastNode?.endLine ?? 0;
-                    const col = lastNode?.endCol ?? 0;
-                    const lastTextNode = {
-                        uuid: uuid(),
-                        raw: lastTextContent,
-                        startOffset: lastOffset,
-                        endOffset: lastOffset + lastTextContent.length,
-                        startLine: line,
-                        endLine: getEndLine(lastTextContent, line),
-                        startCol: col,
-                        endCol: getEndCol(lastTextContent, col),
-                        nodeName: '#text',
-                        type: 'text',
-                        parentNode: null,
-                        prevNode: lastNode,
-                        nextNode: null,
-                        isFragment: false,
-                        isGhost: false,
-                    };
-                    lastNode.nextNode = lastTextNode;
-                    if ((lastNode.type === 'starttag' || lastNode.type === 'endtag') && lastNode.pearNode) {
-                        lastNode.pearNode.nextNode = lastTextNode;
-                    }
-                    nodeOrders.push(lastTextNode);
-                }
-            }
-        }
-    }
-    /**
-     * concat text nodes
-     */
-    const result = [];
-    for (const node of nodeOrders) {
-        const prevNode = result.at(-1) ?? null;
-        if (node.type === 'text' && prevNode?.type === 'text') {
-            prevNode.raw = prevNode.raw + node.raw;
-            prevNode.endOffset = node.endOffset;
-            prevNode.endLine = node.endLine;
-            prevNode.endCol = node.endCol;
-            prevNode.nextNode = node.nextNode;
-            if (prevNode.parentNode) {
-                if (prevNode.parentNode.childNodes) {
-                    if (prevNode.parentNode.childNodes.findIndex(currentChild => currentChild.uuid === prevNode.uuid) === -1) {
-                        prevNode.parentNode.childNodes.unshift(prevNode);
-                    }
-                    else {
-                        prevNode.parentNode.childNodes = [prevNode];
-                    }
-                }
-                prevNode.parentNode.childNodes = prevNode.parentNode.childNodes?.filter(n => n.uuid !== node.uuid);
-            }
-            if (node.nextNode) {
-                node.nextNode.prevNode = prevNode;
-            }
-            continue;
-        }
-        result.push(node);
-    }
-    {
-        /**
-         * Correction prev/next/parent
-         */
-        let prevToken = null;
-        for (const node of result) {
-            if (!prevToken) {
-                prevToken = node;
-                continue;
-            }
-            if (((prevToken.type === 'endtag' && prevToken.nodeName.toLowerCase() === 'body') ||
-                prevToken.type === 'doctype') &&
-                node.type === 'text') {
-                const nextNode = prevToken.nextNode;
-                prevToken.nextNode = node;
-                if (prevToken.type === 'endtag' && prevToken.pearNode) {
-                    prevToken.pearNode.nextNode = node;
-                }
-                node.prevNode = prevToken;
-                node.nextNode = nextNode;
-                node.parentNode = prevToken.parentNode;
-            }
-            // EndTag
-            if (node.type === 'starttag' && node.pearNode) {
-                const endTag = node.pearNode;
-                endTag.pearNode = node;
-                endTag.prevNode = node.prevNode;
-                endTag.nextNode = node.nextNode;
-            }
-            // Children
-            if (node.type === 'text') {
-                const parent = node.parentNode;
-                if (parent &&
-                    parent.type === 'starttag' &&
-                    parent.nodeName.toLowerCase() === 'html' &&
-                    parent.childNodes &&
-                    !parent.childNodes.some(n => n.uuid === node.uuid)) {
-                    parent.childNodes.push(node);
-                }
-            }
-            prevToken = node;
-        }
-    }
-    // console.log(nodeOrders.map((n, i) => `${i}: ${n.raw.trim()}`));
-    return result;
-}
-function arrayize(
-// eslint-disable-next-line @typescript-eslint/prefer-readonly-parameter-types
-nodeTree, rawHtml) {
-    const nodeOrders = [];
-    let prevLine = 1;
-    let prevCol = 1;
-    let currentEndOffset = 0;
-    /**
-     * pushing list
-     */
-    walk(nodeTree, node => {
-        const diff = node.startOffset - currentEndOffset;
-        if (diff > 0) {
-            const html = rawHtml.slice(currentEndOffset, node.startOffset);
-            /**
-             * first white spaces
-             */
-            if (/^\s+$/.test(html)) {
-                const spaces = html;
-                const textNode = {
-                    uuid: uuid(),
-                    raw: spaces,
-                    startOffset: currentEndOffset,
-                    endOffset: currentEndOffset + spaces.length,
-                    startLine: prevLine,
-                    endLine: getEndLine(spaces, prevLine),
-                    startCol: prevCol,
-                    endCol: getEndCol(spaces, prevCol),
-                    nodeName: '#text',
-                    type: 'text',
-                    parentNode: node.parentNode,
-                    prevNode: node.prevNode,
-                    nextNode: node,
-                    isFragment: false,
-                    isGhost: false,
-                };
-                node.prevNode = textNode;
-                if (node.parentNode && node.parentNode.childNodes) {
-                    const newChildNodes = [...node.parentNode.childNodes];
-                    if (newChildNodes.some(child => {
-                        return (
-                        // Out of start offset
-                        textNode.endOffset < child.startOffset ||
-                            // Out of end offset
-                            child.endOffset < textNode.startOffset);
-                    })) {
-                        newChildNodes.push(textNode);
-                    }
-                    newChildNodes.sort((a, b) => a.startOffset - b.startOffset);
-                    node.parentNode.childNodes = newChildNodes;
-                }
-                nodeOrders.push(textNode);
-            }
-        }
-        currentEndOffset = node.startOffset + node.raw.length;
-        prevLine = node.endLine;
-        prevCol = node.endCol;
-        // for ghost nodes
-        node.endOffset = node.endOffset ?? currentEndOffset;
-        nodeOrders.push(node);
-    });
-    return [...nodeOrders];
-}

package/lib/get-space-before.d.ts

@@ -1 +0,0 @@
-export declare function getSpaceBefore(offset: number, rawCode: string): import("packages/@markuplint/ml-ast/lib/types.js").MLToken;

package/lib/get-space-before.js

@@ -1,8 +0,0 @@
-import { createTokenFromRawCode } from './create-token.js';
-export function getSpaceBefore(offset, rawCode) {
-    const aboveCode = rawCode.slice(0, offset);
-    const aboveAttrMatched = aboveCode.match(/\s+$/m);
-    const aboveAttrChar = aboveAttrMatched?.[0] ?? '';
-    const spacesBefore = createTokenFromRawCode(aboveAttrChar, offset - aboveAttrChar.length, rawCode);
-    return spacesBefore;
-}

package/lib/parse-attr.d.ts

@@ -1,24 +0,0 @@
-import type { MLASTHTMLAttr } from '@markuplint/ml-ast';
-type ParseAttrOptions = {
-    readonly booleanish?: boolean;
-    readonly valueDelimiters?: readonly ValueDelimiter[];
-    readonly equal?: string;
-};
-type ValueDelimiter = {
-    readonly start: string;
-    readonly end: string;
-};
-export declare const defaultValueDelimiters: readonly ValueDelimiter[];
-export declare function parseAttr(raw: string, offset: number, html: string, options?: ParseAttrOptions): MLASTHTMLAttr;
-export declare function tokenize(raw: string, options?: ParseAttrOptions): {
-    beforeName: string;
-    name: string;
-    afterName: string;
-    equal: string;
-    beforeValue: string;
-    startQuote: string;
-    value: string;
-    endQuote: string;
-    afterAttr: string;
-};
-export {};

package/lib/parse-attr.js

@@ -1,144 +0,0 @@
-import { createTokenFromRawCode, tokenizer, uuid } from './create-token.js';
-export const defaultValueDelimiters = [
-    {
-        start: "'",
-        end: "'",
-    },
-    {
-        start: '"',
-        end: '"',
-    },
-];
-const defaultEqual = '=';
-const spaceRegex = /^\s$/;
-export function parseAttr(raw, offset, html, options) {
-    const tokens = tokenize(raw, options);
-    tokens.beforeName;
-    const attrToken = createTokenFromRawCode(raw, offset, html);
-    const spacesBeforeName = tokenizer(tokens.beforeName, attrToken.startLine, attrToken.startCol, attrToken.startOffset);
-    const name = tokenizer(tokens.name, spacesBeforeName.endLine, spacesBeforeName.endCol, spacesBeforeName.endOffset);
-    const spacesBeforeEqual = tokenizer(tokens.afterName, name.endLine, name.endCol, name.endOffset);
-    const equal = tokenizer(tokens.equal, spacesBeforeEqual.endLine, spacesBeforeEqual.endCol, spacesBeforeEqual.endOffset);
-    const spacesAfterEqual = tokenizer(tokens.beforeValue, equal.endLine, equal.endCol, equal.endOffset);
-    const startQuote = tokenizer(tokens.startQuote, spacesAfterEqual.endLine, spacesAfterEqual.endCol, spacesAfterEqual.endOffset);
-    const value = tokenizer(tokens.value, startQuote.endLine, startQuote.endCol, startQuote.endOffset);
-    const endQuote = tokenizer(tokens.endQuote, value.endLine, value.endCol, value.endOffset);
-    const attr = {
-        type: 'html-attr',
-        uuid: uuid(),
-        raw: attrToken.raw,
-        startOffset: attrToken.startOffset,
-        endOffset: attrToken.endOffset,
-        startLine: attrToken.startLine,
-        endLine: attrToken.endLine,
-        startCol: attrToken.startCol,
-        endCol: attrToken.endCol,
-        spacesBeforeName,
-        name,
-        spacesBeforeEqual,
-        equal,
-        spacesAfterEqual,
-        startQuote,
-        value,
-        endQuote,
-        isDuplicatable: false,
-        nodeName: name.raw,
-        parentNode: null,
-        nextNode: null,
-        prevNode: null,
-        isFragment: false,
-        isGhost: false,
-    };
-    return attr;
-}
-export function tokenize(raw, options) {
-    const valueDelimiters = options?.valueDelimiters ?? defaultValueDelimiters;
-    const equalDelimiter = options?.equal ?? defaultEqual;
-    let state = 'b-name';
-    const charactors = [...raw];
-    let beforeName = '';
-    let name = '';
-    let afterName = '';
-    let equal = '';
-    let valueDelimiter = null;
-    let beforeValue = '';
-    let startQuote = '';
-    let value = '';
-    let endQuote = '';
-    let afterAttr = '';
-    while (charactors.length > 0) {
-        const charactor = charactors.shift();
-        if (state === 'b-name') {
-            if (spaceRegex.test(charactor)) {
-                beforeName += charactor;
-                continue;
-            }
-            name += charactor;
-            state = 'name';
-            continue;
-        }
-        if (state === 'name') {
-            if (equalDelimiter === charactor) {
-                equal = equalDelimiter;
-                state = 'value-start';
-                continue;
-            }
-            if (spaceRegex.test(charactor)) {
-                afterName += charactor;
-                state = 'a-name';
-                continue;
-            }
-            name += charactor;
-            continue;
-        }
-        if (state === 'a-name') {
-            if (equalDelimiter === charactor) {
-                equal = equalDelimiter;
-                state = 'value-start';
-                continue;
-            }
-            if (spaceRegex.test(charactor)) {
-                afterName += charactor;
-                continue;
-            }
-            break;
-        }
-        if (state === 'value-start') {
-            if (spaceRegex.test(charactor)) {
-                beforeValue += charactor;
-                continue;
-            }
-            valueDelimiter = valueDelimiters.find(d => d.start === charactor) ?? null;
-            if (valueDelimiter) {
-                startQuote += valueDelimiter.start;
-            }
-            else {
-                value += beforeValue + charactor;
-                beforeValue = '';
-            }
-            state = 'value';
-            continue;
-        }
-        if (state !== 'value') {
-            throw new Error('ParseError: unknown parse state in the attribute');
-        }
-        value += charactor;
-    }
-    if (valueDelimiter) {
-        const endQuoteIndex = value.lastIndexOf(valueDelimiter.end);
-        endQuote = value.slice(endQuoteIndex, endQuoteIndex + 1);
-        afterAttr = value.slice(endQuoteIndex + 1);
-        value = value.slice(0, endQuoteIndex);
-    }
-    return {
-        beforeName,
-        name,
-        afterName,
-        equal,
-        beforeValue,
-        startQuote,
-        value,
-        endQuote,
-        afterAttr,
-    };
-}

package/lib/remove-deprecated-node.d.ts

@@ -1,7 +0,0 @@
-import type { MLASTNode } from '@markuplint/ml-ast';
-/**
- *
- * @disruptive
- * @param nodeOrders [Disruptive change]
- */
-export declare function removeDeprecatedNode(nodeOrders: MLASTNode[]): void;

package/lib/remove-deprecated-node.js

@@ -1,39 +0,0 @@
-/**
- *
- * @disruptive
- * @param nodeOrders [Disruptive change]
- */
-export function removeDeprecatedNode(
-// eslint-disable-next-line @typescript-eslint/prefer-readonly-parameter-types
-nodeOrders) {
-    /**
-     * sorting
-     */
-    nodeOrders.sort((a, b) => {
-        if (a.isGhost || b.isGhost) {
-            return 0;
-        }
-        return a.startOffset - b.startOffset;
-    });
-    /**
-     * remove duplicated node
-     */
-    const stack = {};
-    const removeIndexes = [];
-    for (const [i, node] of nodeOrders.entries()) {
-        if (node.isGhost) {
-            continue;
-        }
-        const id = `${node.startLine}:${node.startCol}:${node.endLine}:${node.endCol}`;
-        if (stack[id] != null) {
-            removeIndexes.push(i);
-        }
-        stack[id] = i;
-    }
-    let r = nodeOrders.length;
-    while (r-- > 0) {
-        if (removeIndexes.includes(r)) {
-            nodeOrders.splice(r, 1);
-        }
-    }
-}

package/lib/siblings-correction.d.ts

@@ -1,9 +0,0 @@
-import type { MLASTNode } from '@markuplint/ml-ast';
-/**
- * Correct the references to prevNode and nextNode in the order listed.
- *
- * @param nodeList
- * @affects nodeList[].prevNode
- * @affects nodeList[].nextNode
- */
-export declare function siblingsCorrection(nodeList: MLASTNode[]): void;