pantsdown 2.2.7 → 2.2.8

package/README.md

@@ -165,3 +165,5 @@ console.log(html);
 
 Pantsdown is based on [Marked](https://github.com/markedjs/marked). Without their hard work,
 Pantsdown would not exist.
+
+Last synced with Marked [v18.0.7](https://github.com/markedjs/marked/releases/tag/v18.0.7).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pantsdown",
-  "version": "2.2.7",
+  "version": "2.2.8",
   "description": "Markdown to 'GitHub HTML' parser",
   "license": "MIT",
   "author": "wallpants",
@@ -32,7 +32,8 @@
     "typecheck": "tsc",
     "lint": "oxlint .",
     "check": "bun run typecheck && bun run lint",
-    "docs:build": "bun run docs/build.ts"
+    "docs:build": "bun run docs/build.ts",
+    "prepare": "husky"
   },
   "dependencies": {
     "github-slugger": "^2.0.0",
@@ -40,13 +41,14 @@
     "katex": "^0.18.0"
   },
   "devDependencies": {
+    "@commitlint/cli": "^21.2.1",
     "@commitlint/config-conventional": "^21.2.0",
     "@commitlint/cz-commitlint": "^21.2.0",
     "@happy-dom/global-registrator": "^20.11.0",
     "@types/bun": "^1.3.14",
     "@types/katex": "^0.16.8",
     "commitizen": "^4.3.2",
-    "commitlint": "^21.2.1",
+    "husky": "^9.1.7",
     "inquirer": "^12.11.1",
     "oxfmt": "^0.59.0",
     "oxlint": "^1.73.0",

package/src/lexer.ts

@@ -1,10 +1,11 @@
 import { inline } from "./rules/inline.ts";
+import { other } from "./rules/other.ts";
 import { Tokenizer } from "./tokenizer.ts";
 import { type Links, type SourceMap, type Token, type Tokens } from "./types.ts";
 
 export class Lexer {
    private tokenizer: Tokenizer;
-   private inlineQueue: { src: string; tokens: Token[] }[];
+   inlineQueue: { src: string; tokens: Token[] }[];
    private links: Links = {};
 
    tokens: Token[] = [];
@@ -37,7 +38,7 @@ export class Lexer {
          top: true,
       };
 
-      src = src.replace(/\r\n|\r/g, "\n");
+      src = src.replace(other.carriageReturn, "\n");
 
       this.blockTokens(src, this.tokens);
 
@@ -75,17 +76,19 @@ export class Lexer {
    /**
     * Lexing
     */
-   blockTokens(src: string, tokens: Token[]): Token[] {
-      src = src.replace(/^( *)(\t+)/gm, (_, leading: string, tabs: string) => {
-         return leading + "    ".repeat(tabs.length);
-      });
-
+   blockTokens(src: string, tokens: Token[], lastParagraphClipped = false): Token[] {
       let token: Token | undefined;
       let lastToken: Token | undefined;
       let cutSrc;
-      let lastParagraphClipped;
 
+      let srcLength = Infinity;
       while (src) {
+         if (src.length < srcLength) {
+            srcLength = src.length;
+         } else {
+            this.infiniteLoopError(src.charCodeAt(0));
+         }
+
          // newline
          if ((token = this.tokenizer.space(src))) {
             src = src.substring(token.raw.length);
@@ -106,7 +109,7 @@ export class Lexer {
             lastToken = tokens[tokens.length - 1];
             // An indented code block cannot interrupt a paragraph.
             if (lastToken && (lastToken.type === "paragraph" || lastToken.type === "text")) {
-               lastToken.raw += "\n" + token.raw;
+               lastToken.raw += (lastToken.raw.endsWith("\n") ? "" : "\n") + token.raw;
                lastToken.text += "\n" + token.text;
                const lastInline = this.inlineQueue[this.inlineQueue.length - 1];
                if (lastInline) lastInline.src = lastToken.text;
@@ -177,7 +180,7 @@ export class Lexer {
             src = src.substring(token.raw.length);
             lastToken = tokens[tokens.length - 1];
             if (lastToken && (lastToken.type === "paragraph" || lastToken.type === "text")) {
-               lastToken.raw += "\n" + token.raw;
+               lastToken.raw += (lastToken.raw.endsWith("\n") ? "" : "\n") + token.raw;
                lastToken.text += "\n" + token.raw;
                const lastInline = this.inlineQueue[this.inlineQueue.length - 1];
                if (lastInline) lastInline.src = lastToken.text;
@@ -210,7 +213,7 @@ export class Lexer {
          if (this.state.top && (token = this.tokenizer.paragraph(cutSrc))) {
             lastToken = tokens[tokens.length - 1];
             if (lastParagraphClipped && lastToken?.type === "paragraph") {
-               lastToken.raw += "\n" + token.raw;
+               lastToken.raw += (lastToken.raw.endsWith("\n") ? "" : "\n") + token.raw;
                lastToken.text += "\n" + token.text;
                this.inlineQueue.pop();
                const lastInline = this.inlineQueue[this.inlineQueue.length - 1];
@@ -228,7 +231,7 @@ export class Lexer {
             src = src.substring(token.raw.length);
             lastToken = tokens[tokens.length - 1];
             if (lastToken?.type === "text") {
-               lastToken.raw += "\n" + token.raw;
+               lastToken.raw += (lastToken.raw.endsWith("\n") ? "" : "\n") + token.raw;
                lastToken.text += "\n" + token.text;
                this.inlineQueue.pop();
                const lastInline = this.inlineQueue[this.inlineQueue.length - 1];
@@ -240,8 +243,7 @@ export class Lexer {
          }
 
          if (src) {
-            const errMsg = "Infinite loop on byte: " + String(src.charCodeAt(0));
-            throw new Error(errMsg);
+            this.infiniteLoopError(src.charCodeAt(0));
          }
       }
 
@@ -262,42 +264,34 @@ export class Lexer {
 
       // String with links masked to avoid interference with em and strong
       let maskedSrc = src;
-      let match;
       let keepPrevChar, prevChar;
 
       // Mask out reflinks
       const links = Object.keys(this.links);
       if (links.length > 0) {
-         while ((match = inline.reflinkSearch.exec(maskedSrc)) != null) {
-            if (links.includes(match[0].slice(match[0].lastIndexOf("[") + 1, -1))) {
-               maskedSrc =
-                  maskedSrc.slice(0, match.index) +
-                  "[" +
-                  "a".repeat(match[0].length - 2) +
-                  "]" +
-                  maskedSrc.slice(inline.reflinkSearch.lastIndex);
-            }
-         }
-      }
-      // Mask out other blocks
-      while ((match = inline.blockSkip.exec(maskedSrc)) != null) {
-         maskedSrc =
-            maskedSrc.slice(0, match.index) +
-            "[" +
-            "a".repeat(match[0].length - 2) +
-            "]" +
-            maskedSrc.slice(inline.blockSkip.lastIndex);
+         maskedSrc = maskedSrc.replace(inline.reflinkSearch, (match0) =>
+            links.includes(match0.slice(match0.lastIndexOf("[") + 1, -1))
+               ? "[" + "a".repeat(match0.length - 2) + "]"
+               : match0,
+         );
       }
-
       // Mask out escaped characters
-      while ((match = inline.anyPunctuation.exec(maskedSrc)) != null) {
-         maskedSrc =
-            maskedSrc.slice(0, match.index) +
-            "++" +
-            maskedSrc.slice(inline.anyPunctuation.lastIndex);
-      }
+      maskedSrc = maskedSrc.replace(inline.anyPunctuation, "++");
 
+      // Mask out other blocks
+      maskedSrc = maskedSrc.replace(
+         inline.blockSkip,
+         (match0) => "[" + "a".repeat(match0.length - 2) + "]",
+      );
+
+      let srcLength = Infinity;
       while (src) {
+         if (src.length < srcLength) {
+            srcLength = src.length;
+         } else {
+            this.infiniteLoopError(src.charCodeAt(0));
+         }
+
          if (!keepPrevChar) {
             prevChar = "";
          }
@@ -320,13 +314,7 @@ export class Lexer {
          // tag
          if ((token = this.tokenizer.tag(src))) {
             src = src.substring(token.raw.length);
-            lastToken = tokens[tokens.length - 1];
-            if (lastToken && token.type === "text" && lastToken.type === "text") {
-               lastToken.raw += token.raw;
-               lastToken.text += token.text;
-            } else {
-               tokens.push(token);
-            }
+            tokens.push(token);
             continue;
          }
 
@@ -379,7 +367,7 @@ export class Lexer {
          }
 
          // del (gfm)
-         if ((token = this.tokenizer.del(src))) {
+         if ((token = this.tokenizer.del(src, maskedSrc, prevChar))) {
             src = src.substring(token.raw.length);
             tokens.push(token);
             continue;
@@ -420,11 +408,14 @@ export class Lexer {
          }
 
          if (src) {
-            const errMsg = "Infinite loop on byte: " + String(src.charCodeAt(0));
-            throw new Error(errMsg);
+            this.infiniteLoopError(src.charCodeAt(0));
          }
       }
 
       return tokens;
    }
+
+   private infiniteLoopError(byte: number): never {
+      throw new Error("Infinite loop on byte: " + String(byte));
+   }
 }

package/src/parser.ts

@@ -1,16 +1,19 @@
 import { type Pantsdown } from "./pantsdown.ts";
 import { Renderer } from "./renderer.ts";
+import { TextRenderer } from "./text-renderer.ts";
 import { type Token, type Tokens } from "./types.ts";
-import { injectHtmlAttributes } from "./utils.ts";
 
 /**
  * Parsing & Compiling
  */
 export class Parser {
    renderer: Renderer;
+   textRenderer: TextRenderer;
 
    constructor(pantsdown: Pantsdown) {
       this.renderer = new Renderer(pantsdown);
+      this.renderer.parser = this;
+      this.textRenderer = new TextRenderer();
    }
 
    /**
@@ -24,156 +27,75 @@ export class Parser {
 
          switch (token.type) {
             case "space": {
+               out += this.renderer.space(token);
                continue;
             }
             case "hr": {
-               out += this.renderer.hr(token.sourceMap);
+               out += this.renderer.hr(token);
                continue;
             }
             case "heading": {
-               out += this.renderer.heading(
-                  this.parseInline(token.tokens),
-                  token.depth,
-                  token.sourceMap,
-               );
+               out += this.renderer.heading(token);
                continue;
             }
             case "code": {
-               out += this.renderer.code(token.text, token.lang, token.sourceMap);
+               out += this.renderer.code(token);
                continue;
             }
             case "table": {
-               let header = "";
-
-               // header
-               let cell = "";
-               for (let j = 0, len = token.header.length; j < len; j++) {
-                  cell += this.renderer.tablecell(this.parseInline(token.header[j]!.tokens), {
-                     header: true,
-                     align: token.align[j]!,
-                  });
-               }
-               const sourceMapLineStart = token.sourceMap?.[0];
-               header += this.renderer.tablerow(cell, sourceMapLineStart);
-
-               let body = "";
-               for (let j = 0, rowsLen = token.rows.length; j < rowsLen; j++) {
-                  const row = token.rows[j]!;
-
-                  cell = "";
-                  for (let k = 0, rowLen = row.length; k < rowLen; k++) {
-                     cell += this.renderer.tablecell(this.parseInline(row[k]!.tokens), {
-                        header: false,
-                        align: token.align[k]!,
-                     });
-                  }
-
-                  body += this.renderer.tablerow(
-                     cell,
-                     sourceMapLineStart ? sourceMapLineStart + 2 + j : undefined,
-                  );
-               }
-               out += this.renderer.table(header, body);
+               out += this.renderer.table(token);
                continue;
             }
             case "alert": {
-               const body = this.parse(token.tokens);
-               out += this.renderer.alert(body, token);
+               out += this.renderer.alert(token);
                continue;
             }
             case "blockquote": {
-               const body = this.parse(token.tokens);
-               out += this.renderer.blockquote(body);
+               out += this.renderer.blockquote(token);
                continue;
             }
             case "list": {
-               const ordered = token.ordered;
-               const start = token.start;
-               const loose = token.loose;
-               let containsTaskList = false;
-
-               let body = "";
-               for (let j = 0, itemsLen = token.items.length; j < itemsLen; j++) {
-                  const item = token.items[j]!;
-                  const checked = item.checked;
-                  const task = item.task;
-
-                  let itemBody = "";
-                  if (item.task) {
-                     containsTaskList = true;
-                     const checkbox = this.renderer.checkbox(Boolean(checked), [
-                        "task-list-item-checkbox",
-                     ]);
-                     if (loose) {
-                        if (item.tokens.length > 0 && item.tokens[0]!.type === "paragraph") {
-                           item.tokens[0].text = checkbox + " " + item.tokens[0].text;
-                           if (
-                              item.tokens[0].tokens.length > 0 &&
-                              item.tokens[0].tokens[0]!.type === "text"
-                           ) {
-                              item.tokens[0].tokens[0]!.text =
-                                 checkbox + " " + item.tokens[0].tokens[0]!.text;
-                           }
-                        } else {
-                           item.tokens.unshift({
-                              type: "text",
-                              text: checkbox + " ",
-                           } as Tokens["Text"]);
-                        }
-                     } else {
-                        itemBody += checkbox + " ";
-                     }
-                  }
-
-                  itemBody += this.parse(item.tokens, loose);
-                  body += this.renderer.listitem(itemBody, task, Boolean(checked), item.sourceMap);
-               }
-
-               const listClasses: string[] = [];
-               if (containsTaskList) listClasses.push("contains-task-list");
-               out += this.renderer.list(body, ordered, start, listClasses);
+               out += this.renderer.list(token);
+               continue;
+            }
+            case "checkbox": {
+               out += this.renderer.checkbox(token);
                continue;
             }
             case "html": {
-               out += this.renderer.html(
-                  token.text,
-                  token.block,
-                  "sourceMap" in token ? token.sourceMap : undefined,
-               );
+               out += this.renderer.html(token);
                continue;
             }
             case "footnotes": {
-               const body = token.items.reduce((acc, { label, content, sourceMap }) => {
-                  let footnoteItem = `<li id="footnote-${encodeURIComponent(label)}">\n`;
-                  footnoteItem += this.parse(content);
-                  footnoteItem += "</li>\n";
-
-                  footnoteItem = injectHtmlAttributes(footnoteItem, [], sourceMap);
-
-                  return acc + footnoteItem;
-               }, "");
-
-               out += this.renderer.footnotes(token, body);
+               out += this.renderer.footnotes(token);
                continue;
             }
             case "paragraph": {
-               out += this.renderer.paragraph(this.parseInline(token.tokens), token.sourceMap);
+               out += this.renderer.paragraph(token);
                continue;
             }
             case "text": {
-               let textToken = token as Tokens["Text"];
-               let body = textToken.tokens ? this.parseInline(textToken.tokens) : textToken.text;
+               let textToken: Tokens["Text"] = token;
+               let body = this.renderer.text(textToken);
                while (i + 1 < tokens.length && tokens[i + 1]?.type === "text") {
                   textToken = tokens[++i] as Tokens["Text"];
-                  body +=
-                     "\n" +
-                     (textToken.tokens ? this.parseInline(textToken.tokens) : textToken.text);
+                  body += "\n" + this.renderer.text(textToken);
+               }
+               if (top) {
+                  out += this.renderer.paragraph({
+                     type: "paragraph",
+                     raw: body,
+                     text: body,
+                     tokens: [{ type: "text", raw: body, text: body, escaped: true }],
+                     sourceMap: textToken.sourceMap,
+                  });
+               } else {
+                  out += body;
                }
-               out += top ? this.renderer.paragraph(body, textToken.sourceMap) : body;
                continue;
             }
             case "latexBlock": {
-               out += this.renderer.latexBlock(token.text, token.sourceMap);
+               out += this.renderer.latexBlock(token);
                continue;
             }
 
@@ -190,7 +112,7 @@ export class Parser {
    /**
     * Parse Inline Tokens
     */
-   parseInline(tokens: Token[]): string {
+   parseInline(tokens: Token[], renderer: Renderer | TextRenderer = this.renderer): string {
       let out = "";
 
       for (let i = 0, len = tokens.length; i < len; i++) {
@@ -198,51 +120,55 @@ export class Parser {
 
          switch (token.type) {
             case "escape": {
-               out += this.renderer.text(token.text);
+               out += renderer.text(token);
                break;
             }
             case "html": {
-               out += this.renderer.html(token.text, false);
+               out += renderer.html(token);
                break;
             }
             case "link": {
-               out += this.renderer.link(token.href, token.title, this.parseInline(token.tokens));
+               out += renderer.link(token);
                break;
             }
             case "image": {
-               out += this.renderer.image(token.href, token.title, token.text);
+               out += renderer.image(token);
+               break;
+            }
+            case "checkbox": {
+               out += renderer.checkbox(token);
                break;
             }
             case "strong": {
-               out += this.renderer.strong(this.parseInline(token.tokens));
+               out += renderer.strong(token);
                break;
             }
             case "em": {
-               out += this.renderer.em(this.parseInline(token.tokens));
+               out += renderer.em(token);
                break;
             }
             case "footnoteRef": {
-               out += this.renderer.footnoteRef(token);
+               out += renderer.footnoteRef(token);
                break;
             }
             case "codespan": {
-               out += this.renderer.codespan(token.text);
+               out += renderer.codespan(token);
                break;
             }
             case "br": {
-               out += this.renderer.br();
+               out += renderer.br(token);
                break;
             }
             case "del": {
-               out += this.renderer.del(this.parseInline(token.tokens));
+               out += renderer.del(token);
                break;
             }
             case "text": {
-               out += this.renderer.text(token.text);
+               out += renderer.text(token);
                break;
             }
             case "latexInline": {
-               out += this.renderer.latexInline(token.text);
+               out += renderer.latexInline(token);
                break;
             }
             default: {