markdown-parser 0.1.0 → 0.1.1

package/dist/index.d.ts

@@ -41,7 +41,6 @@ type InlineNode = TextNode | CodeSpanNode | HardBreakNode | SoftBreakNode | Stro
 declare class MarkdownParser {
     private splitter;
     private root;
-    private nextLineIndex;
     private nextNodeIndex;
     private referenceDefinitions;
     parse(input: string, options?: {
@@ -50,6 +49,8 @@ declare class MarkdownParser {
     private parseLine;
     private parseReferenceLinkDefinitions;
     private convertInternalBlockToPublicBlock;
+    get experimental_partialNodes(): BlockNode[];
+    private convertInternalBlockToPartialBlock;
 }
 interface TableNode {
     type: "table";
@@ -83,9 +84,7 @@ interface BlockquoteNode {
 type ListNode = {
     type: "list";
     tight: boolean;
-    items: Array<{
-        children: Array<BlockNode>;
-    }>;
+    items: Array<ListItemNode>;
 } & ({
     kind: "ordered";
     start: number;
@@ -93,6 +92,10 @@ type ListNode = {
     kind: "unordered";
     marker: string;
 });
+type ListItemNode = {
+    type: "list-item";
+    children: Array<BlockNode>;
+};
 interface HtmlBlockNode {
     type: "html-block";
     content: string;

package/dist/index.js

@@ -1140,12 +1140,10 @@ const AUTOLINK_REGEX = /^<[A-Za-z][A-Za-z0-9.+-]{1,31}:[^<>\x00-\x20]*>/i;
 class MarkdownParser {
     parse(input, options) {
         const stream = options?.stream ?? false;
-        const lines = this.splitter.split(input, {
+        for (const line of this.splitter.split(input, {
             stream
-        });
-        for (const line of lines){
+        })){
             this.parseLine(line.replace(/\0/g, "\uFFFD"));
-            this.nextLineIndex++;
         }
         // If stream is false, we need to finalize all remaining open blocks before returning the nodes.
         if (!stream) {
@@ -1165,9 +1163,9 @@ class MarkdownParser {
         return nodes;
     }
     parseLine(line) {
-        const lineIndex = this.nextLineIndex;
         // We start from the root node and descend through the rightmost path of the current node until we find the last open node that the current line continues.
         let lastMatchedNode = this.root;
+        const lists = []; // Tracks contiguous list items in an empty line
         while(true){
             if (!("children" in lastMatchedNode)) break;
             // If there are no children in the current node or if the last/rightmost child of the current node is not open, we do not traverse any further, in which case the last matched node is the current node.
@@ -1181,7 +1179,6 @@ class MarkdownParser {
                 if (blockquote !== null) {
                     line = blockquote.content;
                     lastMatchedNode = child;
-                    child.endLineIndex = lineIndex;
                     continue;
                 } else {
                     break;
@@ -1192,8 +1189,13 @@ class MarkdownParser {
                 if (item === undefined || item.isClosed) break;
                 if (isLineEmpty(line)) {
                     const lastItemChild = item.children.at(-1);
+                    lists.push(item);
                     // If the current line is empty and there are no children in the list item, we close the list item.
                     if (lastItemChild === undefined) {
+                        // Mark all contiguous list items as having a pending blank line
+                        for (const list of lists){
+                            list.hasPendingBlankLine = true;
+                        }
                         closeRightmostPath(child);
                         return;
                     } else {
@@ -1221,11 +1223,9 @@ class MarkdownParser {
                     marker,
                     numOfMarkers
                 })) {
-                    child.endLineIndex = lineIndex;
                     child.isClosed = true;
                     return;
                 } else {
-                    child.endLineIndex = lineIndex;
                     child.lines.push(sliceLeadingIndent(line, child.indentLevel));
                     return;
                 }
@@ -1234,11 +1234,9 @@ class MarkdownParser {
             if (child.type === "indented-code-block") {
                 // We continue the indented code block if the current line is indented enough or if it's empty.
                 if (isIndentedCodeLine(line)) {
-                    child.endLineIndex = lineIndex;
                     child.lines.push(sliceLeadingIndent(line, 4));
                     return;
                 } else if (isLineEmpty(line)) {
-                    child.endLineIndex = lineIndex;
                     child.lines.push("");
                     return;
                 } else {
@@ -1249,10 +1247,12 @@ class MarkdownParser {
             if (child.type === "html-block") {
                 // If the HTML block can be interrupted by a blank line and the current line is empty, we mark the HTML block as closed.
                 if (child.canBeInterruptedByBlankLine && isLineEmpty(line)) {
+                    for (const list of lists){
+                        list.hasPendingBlankLine = true;
+                    }
                     child.isClosed = true;
                     return;
                 }
-                child.endLineIndex = lineIndex;
                 child.lines.push(line);
                 // If the HTML block ends with the current line, we mark the HTML block as closed.
                 if (child.endPattern?.test(line.trim())) {
@@ -1265,6 +1265,10 @@ class MarkdownParser {
             if (child.type === "paragraph" || child.type === "table") {
                 // If the current line is empty, we mark the paragraph or table node as closed and exit as the line has been fully processed and doesn't require any further processing.
                 if (isLineEmpty(line)) {
+                    // If the parent of the paragraph or table node is a list item, we mark it as having a pending blank line.
+                    for (const list of lists){
+                        list.hasPendingBlankLine = true;
+                    }
                     child.isClosed = true;
                     return;
                 } else {
@@ -1272,6 +1276,20 @@ class MarkdownParser {
                 }
             }
         }
+        // If the last matched node is a list item and it has a pending blank line, we mark the parent list as loose.
+        if (lastMatchedNode.type === "list-item" && lastMatchedNode.hasPendingBlankLine) {
+            lastMatchedNode.parent.isTight = false;
+            let node = lastMatchedNode;
+            while(node !== null){
+                if (node.type === "list-item") {
+                    node.hasPendingBlankLine = false;
+                }
+                node = node.parent;
+            }
+        }
+        for (const list of lists){
+            list.hasPendingBlankLine = true;
+        }
         // Represents the node where newly matched nodes are expected to be added. This may not always be true for exceptional cases, like paragraph continuation.
         let currentContainer = lastMatchedNode;
         while(true){
@@ -1282,9 +1300,7 @@ class MarkdownParser {
                     type: "blockquote",
                     children: [],
                     isClosed: false,
-                    parent: currentContainer,
-                    startLineIndex: lineIndex,
-                    endLineIndex: lineIndex
+                    parent: currentContainer
                 };
                 addNode(node);
                 currentContainer = node;
@@ -1299,9 +1315,7 @@ class MarkdownParser {
                     level: heading.level,
                     content: heading.content,
                     isClosed: true,
-                    parent: currentContainer,
-                    startLineIndex: lineIndex,
-                    endLineIndex: lineIndex
+                    parent: currentContainer
                 });
                 break;
             }
@@ -1316,9 +1330,7 @@ class MarkdownParser {
                     info: fencedCode.info,
                     lines: [],
                     isClosed: false,
-                    parent: currentContainer,
-                    startLineIndex: lineIndex,
-                    endLineIndex: lineIndex
+                    parent: currentContainer
                 });
                 break;
             }
@@ -1333,9 +1345,7 @@ class MarkdownParser {
                         line
                     ],
                     isClosed: false,
-                    parent: currentContainer,
-                    startLineIndex: lineIndex,
-                    endLineIndex: lineIndex
+                    parent: currentContainer
                 };
                 if (htmlBlock.canInterruptParagraph) {
                     addNode(node);
@@ -1380,30 +1390,25 @@ class MarkdownParser {
                             rows: []
                         },
                         isClosed: false,
-                        parent: currentContainer,
-                        startLineIndex: lineIndex,
-                        endLineIndex: lineIndex
+                        parent: currentContainer
                     });
                     break;
                 }
                 // Setext heading
                 const heading = parseSetextHeading(line);
                 if (heading !== null) {
-                    let lines = lastChild.lines;
-                    const originalNumOfLines = lines.length;
                     // Parse link reference definitions (if any) from the paragraph lines
-                    let definition = parseLinkReferenceDefinition(lines);
-                    while(definition !== null){
-                        const { label, href, title } = definition.definition;
+                    const result = parseLinkReferenceDefinitions(lastChild.lines);
+                    for (const definition of result.definitions){
+                        const { label, href, title } = definition;
                         if (!this.referenceDefinitions.has(label)) {
                             this.referenceDefinitions.set(label, {
                                 href,
                                 title
                             });
                         }
-                        lines = lines.slice(definition.nextLineIndex);
-                        definition = parseLinkReferenceDefinition(lines);
                     }
+                    const lines = lastChild.lines.slice(result.nextLineIndex);
                     // If there are remaining lines after parsing link reference definitions, we create a new heading node with the remaining lines
                     if (lines.length > 0) {
                         // Remove the current paragraph node from the container node as it'll replaced by a heading node
@@ -1413,9 +1418,7 @@ class MarkdownParser {
                             level: heading.level,
                             content: lines.join("\n").trim(),
                             isClosed: true,
-                            parent: currentContainer,
-                            startLineIndex: lastChild.startLineIndex - (originalNumOfLines - lines.length),
-                            endLineIndex: lineIndex
+                            parent: currentContainer
                         });
                         break;
                     } else {
@@ -1429,9 +1432,7 @@ class MarkdownParser {
                 addNode({
                     type: "thematic-break",
                     isClosed: true,
-                    parent: currentContainer,
-                    startLineIndex: lineIndex,
-                    endLineIndex: lineIndex
+                    parent: currentContainer
                 });
                 break;
             }
@@ -1442,7 +1443,6 @@ class MarkdownParser {
                 const lastChild = currentContainer.children.at(-1);
                 if (lastChild?.type === "paragraph" && !lastChild.isClosed) {
                     if (isLineEmpty(list.content) || list.kind === "ordered" && list.value !== 1) {
-                        lastChild.endLineIndex = lineIndex;
                         lastChild.lines.push(line);
                         break;
                     }
@@ -1461,10 +1461,8 @@ class MarkdownParser {
                             numOfColumns: list.numOfColumns,
                             children: [],
                             isClosed: false,
-                            parent: currentContainer,
-                            startLineIndex: lineIndex,
-                            endLineIndex: lineIndex,
-                            isTight: true
+                            isTight: true,
+                            parent: currentContainer
                         };
                         addNode(parent);
                     }
@@ -1479,23 +1477,24 @@ class MarkdownParser {
                             marker: list.marker,
                             children: [],
                             isClosed: false,
-                            parent: currentContainer,
                             numOfColumns: list.numOfColumns,
-                            startLineIndex: lineIndex,
-                            endLineIndex: lineIndex,
-                            isTight: true
+                            isTight: true,
+                            parent: currentContainer
                         };
                         addNode(parent);
                     }
                 }
+                // If the last child of the parent list is marked as having a pending blank line, we mark this list as loose (i.e., not tight)
+                if (parent.children.at(-1)?.hasPendingBlankLine) {
+                    parent.isTight = false;
+                }
                 parent.numOfColumns = list.numOfColumns;
                 const item = {
                     type: "list-item",
                     children: [],
                     isClosed: false,
-                    parent: parent,
-                    startLineIndex: lineIndex,
-                    endLineIndex: lineIndex
+                    hasPendingBlankLine: false,
+                    parent: parent
                 };
                 addNode(item);
                 currentContainer = item;
@@ -1511,9 +1510,7 @@ class MarkdownParser {
                         sliceLeadingIndent(line, 4)
                     ],
                     isClosed: false,
-                    parent: currentContainer,
-                    startLineIndex: lineIndex,
-                    endLineIndex: lineIndex
+                    parent: currentContainer
                 });
                 break;
             }
@@ -1523,7 +1520,6 @@ class MarkdownParser {
             }
             // Table continuation
             if (lastChild?.type === "table" && !lastChild.isClosed) {
-                lastChild.endLineIndex = lineIndex;
                 const numOfColumns = lastChild.head.cells.length;
                 let cells = parseTableRow(line);
                 if (cells.length > numOfColumns) {
@@ -1543,7 +1539,6 @@ class MarkdownParser {
             }
             // Paragraph
             if (latestOpenNode?.type === "paragraph") {
-                latestOpenNode.endLineIndex = lineIndex;
                 latestOpenNode.lines.push(line);
                 break;
             } else {
@@ -1553,9 +1548,7 @@ class MarkdownParser {
                         line
                     ],
                     isClosed: false,
-                    parent: currentContainer,
-                    startLineIndex: lineIndex,
-                    endLineIndex: lineIndex
+                    parent: currentContainer
                 });
                 break;
             }
@@ -1568,19 +1561,17 @@ class MarkdownParser {
             // If a block is not closed/finalized yet, we do not parse reference link definitions for it
             if (!block.isClosed) break;
             if (block.type === "paragraph") {
-                let lines = block.lines;
-                let definition = parseLinkReferenceDefinition(lines);
-                while(definition !== null){
-                    const { label, href, title } = definition.definition;
+                const result = parseLinkReferenceDefinitions(block.lines);
+                for (const definition of result.definitions){
+                    const { label, href, title } = definition;
                     if (!this.referenceDefinitions.has(label)) {
                         this.referenceDefinitions.set(label, {
                             href,
                             title
                         });
                     }
-                    lines = lines.slice(definition.nextLineIndex);
-                    definition = parseLinkReferenceDefinition(lines);
                 }
+                const lines = block.lines.slice(result.nextLineIndex);
                 if (lines.length > 0) {
                     block.lines = lines;
                 } else {
@@ -1659,6 +1650,7 @@ class MarkdownParser {
             case "list":
                 {
                     const items = block.children.map((item)=>({
+                            type: "list-item",
                             children: item.children.map((child)=>this.convertInternalBlockToPublicBlock(child))
                         }));
                     if (block.kind === "ordered") {
@@ -1705,6 +1697,106 @@ class MarkdownParser {
                 }
         }
     }
+    get experimental_partialNodes() {
+        return this.root.children.map((child)=>this.convertInternalBlockToPartialBlock(child)).filter((child)=>child !== null);
+    }
+    convertInternalBlockToPartialBlock(block) {
+        switch(block.type){
+            case "paragraph":
+                {
+                    if (!block.isClosed) return null;
+                    return this.convertInternalBlockToPublicBlock(block);
+                }
+            case "heading":
+                {
+                    return this.convertInternalBlockToPublicBlock(block);
+                }
+            case "thematic-break":
+                {
+                    return this.convertInternalBlockToPublicBlock(block);
+                }
+            case "fenced-code-block":
+                {
+                    let content = "";
+                    if (block.isClosed) {
+                        content = block.lines.length > 0 ? block.lines.join("\n") + "\n" : "";
+                    } else {
+                        content = block.lines.length > 0 ? block.lines.join("\n") : "";
+                    }
+                    return {
+                        type: "code-block",
+                        content: content,
+                        info: block.info
+                    };
+                }
+            case "indented-code-block":
+                {
+                    let startIndex = 0;
+                    let endIndex = block.lines.length - 1;
+                    while(startIndex < endIndex){
+                        const line = block.lines[startIndex];
+                        if (line === undefined) break;
+                        if (!isLineEmpty(line)) break;
+                        startIndex++;
+                    }
+                    while(endIndex > startIndex){
+                        const line = block.lines[endIndex];
+                        if (line === undefined) break;
+                        if (!isLineEmpty(line)) break;
+                        endIndex--;
+                    }
+                    let content = "";
+                    if (block.isClosed) {
+                        content = block.lines.slice(startIndex, endIndex + 1).join("\n") + "\n";
+                    } else {
+                        content = block.lines.slice(startIndex, endIndex + 1).join("\n");
+                    }
+                    return {
+                        type: "code-block",
+                        content: content
+                    };
+                }
+            case "html-block":
+                {
+                    return this.convertInternalBlockToPublicBlock(block);
+                }
+            case "table":
+                {
+                    return this.convertInternalBlockToPublicBlock(block);
+                }
+            case "blockquote":
+                {
+                    return {
+                        type: "blockquote",
+                        children: block.children.map((child)=>this.convertInternalBlockToPartialBlock(child)).filter((child)=>child !== null)
+                    };
+                }
+            case "list":
+                {
+                    const items = block.children.map((item)=>({
+                            type: "list-item",
+                            children: item.children.map((child)=>this.convertInternalBlockToPartialBlock(child)).filter((child)=>child !== null)
+                        }));
+                    if (block.kind === "ordered") {
+                        return {
+                            type: "list",
+                            kind: "ordered",
+                            start: block.start,
+                            tight: block.isTight,
+                            items: items
+                        };
+                    } else {
+                        return {
+                            type: "list",
+                            kind: "unordered",
+                            marker: block.marker,
+                            tight: block.isTight,
+                            items: items
+                        };
+                    }
+                }
+        }
+    }
     constructor(){
         this.splitter = new LineSplitter();
         this.root = {
@@ -1712,17 +1804,13 @@ class MarkdownParser {
             children: [],
             parent: null
         };
-        this.nextLineIndex = 0;
         this.nextNodeIndex = 0;
         this.referenceDefinitions = new Map();
     }
 }
 function addNode(node) {
-    // Ensure we don't have any dangling "open" nodes on the last-child chain before inserting a new sibling at this level.
     closeRightmostPath(node.parent);
-    // Insert the new node as the next child of the parent.
     if (node.type === "list-item") {
-        // This 'if' condition exists only for TypeScript's type narrowing purposes.
         node.parent.children.push(node);
     } else {
         node.parent.children.push(node);
@@ -1738,61 +1826,20 @@ function addNode(node) {
  * @param parent The parent/container node whose last-child chain will be traversed and closed.
  * @param options.endLineIndex The end line index to update the nodes with.
  */ function closeRightmostPath(parent) {
-    // Find the deepest open node on the rightmost path of the parent container. Then, we close the node, traverse up the parent chain, closing the parent nodes until we reach the root node.
-    // You'll notice that this is not the most efficient way to do this, because we basically travel to the leaf from the parent and then back up to the parent. But, it is required for list tightness calculation.
-    // We could definitely optimize this by keeping track of the deepest open node from parsing the last line, but that would require a lot of additional state management, which is not worth the complexity for now.
-    let currentNode = getDeepestOpenNodeOnRightmostPath(parent);
-    while(currentNode !== parent){
-        if (currentNode === null) break;
-        currentNode.isClosed = true;
-        if (currentNode.type === "list") {
-            currentNode.isTight = isListTight(currentNode);
-        }
-        switch(currentNode.type){
-            case "blockquote":
-            case "list":
-            case "list-item":
-                {
-                    // When closing a container node, we update the start and end line indices based on their first and last child's start and end line indices.
-                    const firstChild = currentNode.children.at(0);
-                    const lastChild = currentNode.children.at(-1);
-                    if (firstChild !== undefined) {
-                        currentNode.startLineIndex = Math.min(currentNode.startLineIndex, firstChild.startLineIndex);
-                    }
-                    if (lastChild !== undefined) {
-                        currentNode.endLineIndex = Math.max(currentNode.endLineIndex, lastChild.endLineIndex);
-                    }
-                    break;
-                }
-        }
-        if (currentNode.parent?.type === "root") break;
-        currentNode = currentNode.parent;
-    }
-}
-function isListTight(list) {
-    // 1) Check if there are gaps between list items
-    const items = list.children;
-    for(let i = 0; i < items.length - 1; i++){
-        const a = items[i];
-        const b = items[i + 1];
-        if (a === undefined || b === undefined) break;
-        if (a.endLineIndex < b.startLineIndex - 1) {
-            return false;
-        }
-    }
-    // 2) Check if there are gaps between block children inside each item
-    for (const item of items){
-        const blocks = item.children;
-        for(let j = 0; j < blocks.length - 1; j++){
-            const a = blocks[j];
-            const b = blocks[j + 1];
-            if (a === undefined || b === undefined) break;
-            if (a.endLineIndex < b.startLineIndex - 1) {
-                return false;
-            }
+    // Walk down the rightmost/last-child chain and close any still-open nodes until we hit a closed one (or run out).
+    // Start at the last child of the parent; we'll keep descending into the last child of certain container nodes (e.g., blockquotes).
+    let child = parent.children.at(-1);
+    while(child !== undefined){
+        // If we've hit a node that is already closed, everything below it on this rightmost chain is assumed to be closed too, so we can stop early.
+        if (child.isClosed) break;
+        child.isClosed = true;
+        // If the child is a container node, we continue descending into the last child of the container node.
+        if (child.type === "blockquote" || child.type === "list" || child.type === "list-item") {
+            child = child.children.at(-1);
+        } else {
+            break;
         }
     }
-    return true;
 }
 function getDeepestOpenNodeOnRightmostPath(node) {
     const child = node.children.at(-1);
@@ -2307,10 +2354,28 @@ function parseTableRow(line) {
     }
     return cells;
 }
-function parseLinkReferenceDefinition(lines) {
-    const firstLine = lines[0];
+function parseLinkReferenceDefinitions(lines) {
+    const definitions = [];
+    let nextLineIndex = 0;
+    let definition = _parseLinkReferenceDefinition(lines, {
+        startLineIndex: nextLineIndex
+    });
+    while(definition !== null){
+        definitions.push(definition.definition);
+        nextLineIndex = definition.nextLineIndex;
+        definition = _parseLinkReferenceDefinition(lines, {
+            startLineIndex: nextLineIndex
+        });
+    }
+    return {
+        definitions,
+        nextLineIndex
+    };
+}
+function _parseLinkReferenceDefinition(lines, options) {
+    const firstLine = lines[options.startLineIndex];
     if (firstLine === undefined) return null;
-    let nextLineIndex = 1;
+    let nextLineIndex = options.startLineIndex + 1;
     let content = firstLine.trim() + "\n";
     // A link reference definition must start with a left bracket '['
     if (content.charAt(0) !== "[") return null;

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "markdown-parser",
 	"description": "A streaming-capable markdown parser.",
-	"version": "0.1.0",
+	"version": "0.1.1",
 	"license": "MIT",
 	"type": "module",
 	"sideEffects": false,