@markuplint/parser-utils 4.0.0-dev.0 → 4.0.0-dev.12

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017-2023 Yusuke Hirao
+Copyright (c) 2017-2024 Yusuke Hirao
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/lib/attr-tokenizer.d.ts

@@ -1,6 +1,18 @@
 import type { QuoteSet } from './types.js';
-import type { MLASTHTMLAttr } from '@markuplint/ml-ast';
-import { AttrState } from './attr-parser.js';
-export declare function attrTokenizer(raw: string, line: number, col: number, startOffset: number, quoteSet?: ReadonlyArray<QuoteSet>, startState?: AttrState, quoteInValueChars?: ReadonlyArray<QuoteSet>, spaces?: ReadonlyArray<string>): MLASTHTMLAttr & {
-    __leftover?: string;
+import { AttrState } from './enums.js';
+/**
+ * @see https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
+ * @see https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-name-state
+ * @see https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
+ */
+export declare function attrTokenizer(raw: string, quoteSet?: readonly QuoteSet[], startState?: AttrState, quoteInValueChars?: ReadonlyArray<QuoteSet>, endOfUnquotedValueChars?: ReadonlyArray<string>): {
+    spacesBeforeAttrName: string;
+    attrName: string;
+    spacesBeforeEqual: string;
+    equal: string;
+    spacesAfterEqual: string;
+    quoteStart: string;
+    attrValue: string;
+    quoteEnd: string;
+    leftover: string;
 };

package/lib/attr-tokenizer.js

@@ -1,75 +1,169 @@
-import { AttrState, attrParser } from './attr-parser.js';
-import { tokenizer, uuid } from './create-token.js';
-export function attrTokenizer(raw, line, col, startOffset, quoteSet, startState = AttrState.BeforeName, quoteInValueChars, spaces) {
-    const parsed = attrParser(raw, quoteSet, startState, quoteInValueChars, spaces);
-    let offset = startOffset;
-    const spacesBeforeName = tokenizer(parsed.spacesBeforeAttrName, line, col, offset);
-    line = spacesBeforeName.endLine;
-    col = spacesBeforeName.endCol;
-    offset = spacesBeforeName.endOffset;
-    const name = tokenizer(parsed.attrName, line, col, offset);
-    line = name.endLine;
-    col = name.endCol;
-    offset = name.endOffset;
-    const spacesBeforeEqual = tokenizer(parsed.spacesBeforeEqual, line, col, offset);
-    line = spacesBeforeEqual.endLine;
-    col = spacesBeforeEqual.endCol;
-    offset = spacesBeforeEqual.endOffset;
-    const equal = tokenizer(parsed.equal, line, col, offset);
-    line = equal.endLine;
-    col = equal.endCol;
-    offset = equal.endOffset;
-    const spacesAfterEqual = tokenizer(parsed.spacesAfterEqual, line, col, offset);
-    line = spacesAfterEqual.endLine;
-    col = spacesAfterEqual.endCol;
-    offset = spacesAfterEqual.endOffset;
-    const startQuote = tokenizer(parsed.quoteStart, line, col, offset);
-    line = startQuote.endLine;
-    col = startQuote.endCol;
-    offset = startQuote.endOffset;
-    const value = tokenizer(parsed.attrValue, line, col, offset);
-    line = value.endLine;
-    col = value.endCol;
-    offset = value.endOffset;
-    const endQuote = tokenizer(parsed.quoteEnd, line, col, offset);
-    const attrToken = tokenizer(parsed.attrName +
-        parsed.spacesBeforeEqual +
-        parsed.equal +
-        parsed.spacesAfterEqual +
-        parsed.quoteStart +
-        parsed.attrValue +
-        parsed.quoteEnd, name.startLine, name.startCol, name.startOffset);
-    const result = {
-        type: 'html-attr',
-        uuid: uuid(),
-        raw: attrToken.raw,
-        startOffset: attrToken.startOffset,
-        endOffset: attrToken.endOffset,
-        startLine: attrToken.startLine,
-        endLine: attrToken.endLine,
-        startCol: attrToken.startCol,
-        endCol: attrToken.endCol,
-        spacesBeforeName,
-        name,
+import { defaultSpaces } from './const.js';
+import { AttrState } from './enums.js';
+const defaultQuoteSet = [
+    { start: '"', end: '"' },
+    { start: "'", end: "'" },
+];
+const defaultQuoteInValueChars = [];
+const spaces = defaultSpaces;
+const EQUAL = '=';
+/**
+ * @see https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
+ * @see https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-name-state
+ * @see https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
+ */
+export function attrTokenizer(raw, quoteSet = defaultQuoteSet, startState = AttrState.BeforeName, quoteInValueChars = defaultQuoteInValueChars, endOfUnquotedValueChars = [...defaultSpaces, '/', '>']) {
+    let state = startState;
+    let spacesBeforeAttrName = '';
+    let attrName = '';
+    let spacesBeforeEqual = '';
+    let equal = '';
+    let spacesAfterEqual = '';
+    let quoteTypeIndex = -1;
+    let quoteStart = '';
+    let attrValue = '';
+    let quoteEnd = '';
+    const isBeforeValueStarted = startState === AttrState.BeforeValue;
+    const quoteModeStack = [];
+    const chars = [...raw];
+    while (chars.length > 0) {
+        if (state === AttrState.AfterValue) {
+            break;
+        }
+        const char = chars.shift();
+        switch (state) {
+            case AttrState.BeforeName: {
+                if (char === '>') {
+                    chars.unshift(char);
+                    state = AttrState.AfterValue;
+                    break;
+                }
+                if (char === '/') {
+                    chars.unshift(char);
+                    state = AttrState.AfterValue;
+                    break;
+                }
+                if (spaces.includes(char)) {
+                    spacesBeforeAttrName += char;
+                    break;
+                }
+                attrName += char;
+                state = AttrState.Name;
+                break;
+            }
+            case AttrState.Name: {
+                if (char === '>') {
+                    chars.unshift(char);
+                    state = AttrState.AfterValue;
+                    break;
+                }
+                if (char === '/') {
+                    chars.unshift(char);
+                    state = AttrState.AfterValue;
+                    break;
+                }
+                if (spaces.includes(char)) {
+                    spacesBeforeEqual += char;
+                    state = AttrState.Equal;
+                    break;
+                }
+                if (char === EQUAL) {
+                    equal += char;
+                    state = AttrState.BeforeValue;
+                    break;
+                }
+                attrName += char;
+                break;
+            }
+            case AttrState.Equal: {
+                if (spaces.includes(char)) {
+                    spacesBeforeEqual += char;
+                    break;
+                }
+                if (char === EQUAL) {
+                    equal += char;
+                    state = AttrState.BeforeValue;
+                    break;
+                }
+                // End of attribute
+                chars.unshift(spacesBeforeEqual, char);
+                spacesBeforeEqual = '';
+                state = AttrState.AfterValue;
+                break;
+            }
+            case AttrState.BeforeValue: {
+                if (endOfUnquotedValueChars.includes(char) && spaces.includes(char)) {
+                    if (isBeforeValueStarted) {
+                        spacesBeforeAttrName += char;
+                        break;
+                    }
+                    spacesAfterEqual += char;
+                    break;
+                }
+                quoteTypeIndex = quoteSet.findIndex(quote => quote.start === char);
+                const quote = quoteSet[quoteTypeIndex];
+                if (quote) {
+                    quoteStart = quote.start;
+                    state = AttrState.Value;
+                    break;
+                }
+                const raw = char + chars.join('');
+                const inQuote = quoteInValueChars.find(quote => raw.startsWith(quote.start));
+                if (inQuote) {
+                    quoteModeStack.push(inQuote);
+                    attrValue += inQuote.start;
+                    chars.splice(0, inQuote.start.length - 1);
+                    state = AttrState.Value;
+                    break;
+                }
+                chars.unshift(char);
+                state = AttrState.Value;
+                break;
+            }
+            case AttrState.Value: {
+                if (!quoteSet[quoteTypeIndex] && endOfUnquotedValueChars.includes(char)) {
+                    chars.unshift(char);
+                    state = AttrState.AfterValue;
+                    break;
+                }
+                if (quoteModeStack.length === 0 && char === quoteSet[quoteTypeIndex]?.end) {
+                    quoteEnd = char;
+                    state = AttrState.AfterValue;
+                    break;
+                }
+                const raw = char + chars.join('');
+                const inQuoteEnd = quoteModeStack.at(-1);
+                if (inQuoteEnd && raw.startsWith(inQuoteEnd.end)) {
+                    quoteModeStack.pop();
+                    attrValue += inQuoteEnd.end;
+                    chars.splice(0, inQuoteEnd.end.length - 1);
+                    break;
+                }
+                const inQuoteStart = quoteInValueChars.find(quote => raw.startsWith(quote.start));
+                if (inQuoteStart) {
+                    quoteModeStack.push(inQuoteStart);
+                    attrValue += inQuoteStart.start;
+                    chars.splice(0, inQuoteStart.start.length - 1);
+                    break;
+                }
+                attrValue += char;
+                break;
+            }
+        }
+    }
+    if (state === AttrState.Value && quoteTypeIndex !== -1) {
+        throw new SyntaxError(`Unclosed attribute value: ${raw}`);
+    }
+    const leftover = chars.join('');
+    return {
+        spacesBeforeAttrName,
+        attrName,
         spacesBeforeEqual,
         equal,
         spacesAfterEqual,
-        startQuote,
-        value,
-        endQuote,
-        isDuplicatable: false,
-        nodeName: name.raw,
-        parentNode: null,
-        prevNode: null,
-        nextNode: null,
-        isFragment: false,
-        isGhost: false,
+        quoteStart,
+        attrValue,
+        quoteEnd,
+        leftover,
     };
-    if (parsed.leftover) {
-        return {
-            ...result,
-            __leftover: parsed.leftover,
-        };
-    }
-    return result;
 }

package/lib/const.d.ts

@@ -11,6 +11,7 @@ export declare const reSplitterTag: RegExp;
  * - U+0009 CHARACTER TABULATION (tab) => `\t`
  * - U+000A LINE FEED (LF) => `\n`
  * - U+000C FORM FEED (FF) => `\f`
+ * - U+000D CARRIAGE RETURN (CR) => `\r`
  * - U+0020 SPACE => ` `
  */
-export declare const defaultSpaces: readonly ["\t", "\n", "\f", " "];
+export declare const defaultSpaces: readonly ["\t", "\n", "\f", "\r", " "];

package/lib/const.js

@@ -100,6 +100,7 @@ export const reSplitterTag = /<[^>]+>/g;
  * - U+0009 CHARACTER TABULATION (tab) => `\t`
  * - U+000A LINE FEED (LF) => `\n`
  * - U+000C FORM FEED (FF) => `\f`
+ * - U+000D CARRIAGE RETURN (CR) => `\r`
  * - U+0020 SPACE => ` `
  */
-export const defaultSpaces = ['\t', '\n', '\f', ' '];
+export const defaultSpaces = ['\t', '\n', '\f', '\r', ' '];

package/lib/debugger.d.ts

@@ -1,3 +1,4 @@
 import type { MLASTAttr, MLASTNode } from '@markuplint/ml-ast';
-export declare function nodeListToDebugMaps(nodeList: MLASTNode[], withAttr?: boolean): string[];
-export declare function attributesToDebugMaps(attributes: MLASTAttr[]): string[][];
+export declare function nodeListToDebugMaps(nodeList: readonly (MLASTNode | null)[], withAttr?: boolean): string[];
+export declare function attributesToDebugMaps(attributes: readonly MLASTAttr[]): string[][];
+export declare function nodeTreeDebugView(nodeTree: readonly MLASTNode[]): (string | undefined)[];

package/lib/debugger.js

@@ -1,46 +1,62 @@
-export function nodeListToDebugMaps(
-// eslint-disable-next-line @typescript-eslint/prefer-readonly-parameter-types
-nodeList, withAttr = false) {
+export function nodeListToDebugMaps(nodeList, withAttr = false) {
     return nodeList.flatMap(n => {
         const r = [];
-        if (n.isGhost) {
-            r.push(`[N/A]>[N/A](N/A)${n.nodeName}: ${visibleWhiteSpace(n.raw)}`);
-        }
-        else {
-            r.push(tokenDebug(n));
-            if (withAttr && 'attributes' in n) {
-                r.push(...attributesToDebugMaps(n.attributes).flat());
-            }
+        r.push(tokenDebug(n));
+        if (withAttr && n && n.type === 'starttag') {
+            r.push(...attributesToDebugMaps(n.attributes).flat());
         }
         return r;
     });
 }
-export function attributesToDebugMaps(
-// eslint-disable-next-line @typescript-eslint/prefer-readonly-parameter-types
-attributes) {
+export function attributesToDebugMaps(attributes) {
     return attributes.map(n => {
         const r = [
             tokenDebug({
                 ...n,
-                name: n.type === 'html-attr' ? n.name.raw : n.raw,
+                name: n.type === 'attr' ? n.name.raw : n.raw,
             }),
         ];
-        if (n.type === 'html-attr') {
-            r.push(`  ${tokenDebug(n.spacesBeforeName, 'bN')}`, `  ${tokenDebug(n.name, 'name')}`, `  ${tokenDebug(n.spacesBeforeEqual, 'bE')}`, `  ${tokenDebug(n.equal, 'equal')}`, `  ${tokenDebug(n.spacesAfterEqual, 'aE')}`, `  ${tokenDebug(n.startQuote, 'sQ')}`, `  ${tokenDebug(n.value, 'value')}`, `  ${tokenDebug(n.endQuote, 'eQ')}`, `  isDirective: ${!!n.isDirective}`, `  isDynamicValue: ${!!n.isDynamicValue}`);
+        if (n.type === 'spread') {
+            r.push(`  #spread: ${visibleWhiteSpace(n.raw)}`);
+            return r;
         }
+        r.push(`  ${tokenDebug(n.spacesBeforeName, 'bN')}`, `  ${tokenDebug(n.name, 'name')}`, `  ${tokenDebug(n.spacesBeforeEqual, 'bE')}`, `  ${tokenDebug(n.equal, 'equal')}`, `  ${tokenDebug(n.spacesAfterEqual, 'aE')}`, `  ${tokenDebug(n.startQuote, 'sQ')}`, `  ${tokenDebug(n.value, 'value')}`, `  ${tokenDebug(n.endQuote, 'eQ')}`, `  isDirective: ${!!n.isDirective}`, `  isDynamicValue: ${!!n.isDynamicValue}`);
         if (n.potentialName != null) {
             r.push(`  potentialName: ${visibleWhiteSpace(n.potentialName)}`);
         }
-        if (n.type === 'html-attr' && n.candidate) {
+        if (n.candidate) {
             r.push(`  candidate: ${visibleWhiteSpace(n.candidate)}`);
         }
         return r;
     });
 }
+export function nodeTreeDebugView(nodeTree) {
+    return nodeTree
+        .map((n, i) => {
+        const lines = [];
+        if (n.type === 'attr' || n.type === 'spread') {
+            return;
+        }
+        lines.push(`${i.toString().padStart(3, '0')}: [${n.uuid.slice(0, 8)}] ${'  '.repeat(Math.max(n.depth, 0))}${n.type === 'endtag' ? '/' : ''}${n.nodeName}(${n.uuid.slice(0, 8)})${n.type === 'starttag' && n.isGhost ? '[👻]' : ''}${n.type === 'starttag'
+            ? ` => ${n.pairNode ? `/${n.pairNode.nodeName}(${n.pairNode.uuid.slice(0, 8)})` : '💀'}`
+            : ''}`);
+        if (n.type === 'starttag' || n.type === 'psblock') {
+            for (const c of n.childNodes ?? []) {
+                lines.push(`${' '.repeat(15)}   ${'  '.repeat(Math.max(n.depth, 0))}┗━ ${c.type === 'endtag' ? '/' : ''}${c.nodeName}(${c.uuid.slice(0, 8)})`);
+            }
+        }
+        return lines;
+    })
+        .filter(Boolean)
+        .flat();
+}
 function tokenDebug(n, type = '') {
+    if (!n) {
+        return 'NULL';
+    }
     return `[${n.startLine}:${n.startCol}]>[${n.endLine}:${n.endCol}](${n.startOffset},${n.endOffset})${
     // @ts-ignore
-    n.potentialName ?? n.nodeName ?? n.name ?? n.type ?? type}: ${visibleWhiteSpace(n.raw)}`;
+    n.potentialName ?? n.nodeName ?? n.name ?? n.type ?? type}${'isGhost' in n && n.isGhost ? '(👻)' : ''}${'isBogus' in n && n.isBogus ? '(👿)' : ''}: ${visibleWhiteSpace(n.raw)}`;
 }
 function visibleWhiteSpace(chars) {
     return chars.replaceAll('\n', '⏎').replaceAll('\t', '→').replaceAll(/\s/g, '␣');

package/lib/enums.d.ts

@@ -0,0 +1,16 @@
+export declare enum TagState {
+    BeforeOpenTag = 0,
+    FirstCharOfTagName = 1,
+    TagName = 2,
+    Attrs = 3,
+    AfterAttrs = 4,
+    AfterOpenTag = 5
+}
+export declare enum AttrState {
+    BeforeName = 0,
+    Name = 1,
+    Equal = 2,
+    BeforeValue = 3,
+    Value = 4,
+    AfterValue = 5
+}

package/lib/enums.js

@@ -0,0 +1,18 @@
+export var TagState;
+(function (TagState) {
+    TagState[TagState["BeforeOpenTag"] = 0] = "BeforeOpenTag";
+    TagState[TagState["FirstCharOfTagName"] = 1] = "FirstCharOfTagName";
+    TagState[TagState["TagName"] = 2] = "TagName";
+    TagState[TagState["Attrs"] = 3] = "Attrs";
+    TagState[TagState["AfterAttrs"] = 4] = "AfterAttrs";
+    TagState[TagState["AfterOpenTag"] = 5] = "AfterOpenTag";
+})(TagState || (TagState = {}));
+export var AttrState;
+(function (AttrState) {
+    AttrState[AttrState["BeforeName"] = 0] = "BeforeName";
+    AttrState[AttrState["Name"] = 1] = "Name";
+    AttrState[AttrState["Equal"] = 2] = "Equal";
+    AttrState[AttrState["BeforeValue"] = 3] = "BeforeValue";
+    AttrState[AttrState["Value"] = 4] = "Value";
+    AttrState[AttrState["AfterValue"] = 5] = "AfterValue";
+})(AttrState || (AttrState = {}));

package/lib/get-location.d.ts

@@ -1,13 +1,4 @@
-export declare function getLine(html: string, startOffset: number): number;
-export declare function getCol(html: string, startOffset: number): number;
-export declare function getEndLine(html: string, line: number): number;
-export declare function getEndCol(html: string, col: number): number;
-export declare function sliceFragment(rawHtml: string, start: number, end: number): {
-    startOffset: number;
-    endOffset: number;
-    startLine: number;
-    endLine: number;
-    startCol: number;
-    endCol: number;
-    raw: string;
-};
+export declare function getLine(rawCodeFragment: string, startOffset: number): number;
+export declare function getCol(rawCodeFragment: string, startOffset: number): number;
+export declare function getEndLine(rawCodeFragment: string, startLine: number): number;
+export declare function getEndCol(rawCodeFragment: string, startCol: number): number;

package/lib/get-location.js

@@ -1,28 +1,17 @@
-export function getLine(html, startOffset) {
-    return html.slice(0, startOffset).split(/\n/).length;
+const LINE_BREAK = '\n';
+export function getLine(rawCodeFragment, startOffset) {
+    return rawCodeFragment.slice(0, startOffset).split(LINE_BREAK).length;
 }
-export function getCol(html, startOffset) {
-    const lines = html.slice(0, startOffset).split(/\n/);
+export function getCol(rawCodeFragment, startOffset) {
+    const lines = rawCodeFragment.slice(0, startOffset).split(LINE_BREAK);
     return (lines.at(-1) ?? '').length + 1;
 }
-export function getEndLine(html, line) {
-    return html.split(/\r?\n/).length - 1 + line;
+export function getEndLine(rawCodeFragment, startLine) {
+    return rawCodeFragment.split(LINE_BREAK).length - 1 + startLine;
 }
-export function getEndCol(html, col) {
-    const lines = html.split(/\r?\n/);
+export function getEndCol(rawCodeFragment, startCol) {
+    const lines = rawCodeFragment.split(LINE_BREAK);
     const lineCount = lines.length;
     const lastLine = lines.pop();
-    return lineCount > 1 ? lastLine.length + 1 : col + html.length;
-}
-export function sliceFragment(rawHtml, start, end) {
-    const raw = rawHtml.slice(start, end);
-    return {
-        startOffset: start,
-        endOffset: end,
-        startLine: getLine(rawHtml, start),
-        endLine: getLine(rawHtml, end),
-        startCol: getCol(rawHtml, start),
-        endCol: getCol(rawHtml, end),
-        raw,
-    };
+    return lineCount > 1 ? lastLine.length + 1 : startCol + rawCodeFragment.length;
 }

package/lib/ignore-block.d.ts

@@ -1,4 +1,5 @@
+import type { Parser } from './parser.js';
 import type { IgnoreBlock, IgnoreTag } from './types.js';
-import type { MLASTNode } from '@markuplint/ml-ast';
+import type { MLASTNodeTreeItem } from '@markuplint/ml-ast';
 export declare function ignoreBlock(source: string, tags: readonly IgnoreTag[], maskChar?: string): IgnoreBlock;
-export declare function restoreNode(nodeList: MLASTNode[], ignoreBlock: IgnoreBlock): MLASTNode[];
+export declare function restoreNode(parser: Parser<any, any>, nodeList: readonly MLASTNodeTreeItem[], ignoreBlock: IgnoreBlock, throwErrorWhenTagHasUnresolved?: boolean): MLASTNodeTreeItem[];