pantsdown 1.0.1 → 1.1.1

package/.github/workflows/release.yaml

@@ -1,5 +1,6 @@
 name: Release
 on:
+    pull_request:
     push:
         branches:
             - main

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "pantsdown",
-  "version": "1.0.1",
+  "version": "1.1.1",
+  "author": "wallpants",
   "description": "The markdown parser",
+  "license": "MIT",
+  "type": "module",
   "repository": {
     "type": "git",
     "url": "git://github.com/wallpants/pantsdown.git"
   },
-  "license": "MIT",
-  "type": "module",
   "exports": {
     "types": "./src/index.ts",
     "bun": "./src/index.ts",
@@ -28,6 +29,9 @@
       "main"
     ]
   },
+  "dependencies": {
+    "highlight.js": "^11.9.0"
+  },
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^6.7.3",
     "@typescript-eslint/parser": "^6.7.3",
@@ -39,8 +43,5 @@
     "prettier-plugin-organize-imports": "^3.2.3",
     "semantic-release": "^22.0.5",
     "typescript": "5.2.2"
-  },
-  "dependencies": {
-    "highlight.js": "^11.9.0"
   }
 }

package/src/lexer.ts

@@ -1,12 +1,13 @@
 import { inline } from "./rules/inline.ts";
 import { Tokenizer } from "./tokenizer.ts";
-import { type Links, type Token } from "./types.ts";
+import { type Links, type SourceMap, type Token } from "./types.ts";
 
 export class Lexer {
     private tokenizer: Tokenizer;
     private inlineQueue: { src: string; tokens: Token[] }[];
     private tokens: Token[] = [];
     private links: Links = {};
+    private line = 1;
 
     state = {
         inLink: false,
@@ -26,6 +27,7 @@ export class Lexer {
         // reset values from previous parse
         this.tokens = [];
         this.links = {};
+        this.line = 1;
 
         src = src.replace(/\r\n|\r/g, "\n");
 
@@ -39,6 +41,21 @@ export class Lexer {
         return this.tokens;
     }
 
+    getSourceMap(rawToken: string): SourceMap | undefined {
+        if (!this.state.top) return;
+        const tokenLines = rawToken.split("\n");
+        const sourceMap: SourceMap = [this.line, (this.line += tokenLines.length - 1)];
+
+        while (tokenLines.slice(-1)[0] === "") {
+            // token.raw sometimes includes newline chars at the end of the string
+            // which we don't want to include in the sourceMap
+            tokenLines.pop();
+            sourceMap[1]--;
+        }
+
+        return sourceMap;
+    }
+
     /**
      * Lexing
      */

package/src/parser.ts

@@ -25,15 +25,19 @@ export class Parser {
                     continue;
                 }
                 case "hr": {
-                    out += this.renderer.hr();
+                    out += this.renderer.hr(token.sourceMap);
                     continue;
                 }
                 case "heading": {
-                    out += this.renderer.heading(this.parseInline(token.tokens), token.depth);
+                    out += this.renderer.heading(
+                        this.parseInline(token.tokens),
+                        token.depth,
+                        token.sourceMap,
+                    );
                     continue;
                 }
                 case "code": {
-                    out += this.renderer.code(token.text, token.lang);
+                    out += this.renderer.code(token.text, token.lang, token.sourceMap);
                     continue;
                 }
                 case "table": {
@@ -47,7 +51,8 @@ export class Parser {
                             align: token.align[j]!,
                         });
                     }
-                    header += this.renderer.tablerow(cell);
+                    const sourceMapLineStart = token.sourceMap?.[0];
+                    header += this.renderer.tablerow(cell, sourceMapLineStart);
 
                     let body = "";
                     for (let j = 0, rowsLen = token.rows.length; j < rowsLen; j++) {
@@ -61,14 +66,17 @@ export class Parser {
                             });
                         }
 
-                        body += this.renderer.tablerow(cell);
+                        body += this.renderer.tablerow(
+                            cell,
+                            sourceMapLineStart ? sourceMapLineStart + 2 + j : undefined,
+                        );
                     }
-                    out += this.renderer.table(header, body);
+                    out += this.renderer.table(header, body, token.sourceMap);
                     continue;
                 }
                 case "blockquote": {
                     const body = this.parse(token.tokens);
-                    out += this.renderer.blockquote(body);
+                    out += this.renderer.blockquote(body, token.sourceMap);
                     continue;
                 }
                 case "list": {
@@ -84,7 +92,7 @@ export class Parser {
 
                         let itemBody = "";
                         if (item.task) {
-                            const checkbox = this.renderer.checkbox(!!checked);
+                            const checkbox = this.renderer.checkbox(Boolean(checked));
                             if (loose) {
                                 if (
                                     item.tokens.length > 0 &&
@@ -110,18 +118,22 @@ export class Parser {
                         }
 
                         itemBody += this.parse(item.tokens, loose);
-                        body += this.renderer.listitem(itemBody, task, !!checked);
+                        body += this.renderer.listitem(itemBody, task, Boolean(checked));
                     }
 
-                    out += this.renderer.list(body, ordered, start);
+                    out += this.renderer.list(body, ordered, start, token.sourceMap);
                     continue;
                 }
                 case "html": {
-                    out += this.renderer.html(token.text, token.block);
+                    out += this.renderer.html(
+                        token.text,
+                        token.block,
+                        "sourceMap" in token ? token.sourceMap : undefined,
+                    );
                     continue;
                 }
                 case "paragraph": {
-                    out += this.renderer.paragraph(this.parseInline(token.tokens));
+                    out += this.renderer.paragraph(this.parseInline(token.tokens), token.sourceMap);
                     continue;
                 }
                 case "text": {
@@ -137,7 +149,7 @@ export class Parser {
                                 ? this.parseInline(textToken.tokens)
                                 : textToken.text);
                     }
-                    out += top ? this.renderer.paragraph(body) : body;
+                    out += top ? this.renderer.paragraph(body, textToken.sourceMap) : body;
                     continue;
                 }
 
@@ -166,7 +178,7 @@ export class Parser {
                     break;
                 }
                 case "html": {
-                    out += this.renderer.html(token.text);
+                    out += this.renderer.html(token.text, false);
                     break;
                 }
                 case "link": {

package/src/renderer.ts

@@ -1,6 +1,13 @@
 import hljs from "highlight.js";
-import { type RendererProps } from "./types.ts";
-import { cleanUrl, escape, fixHtmlLocalImageHref, fixLocalImageHref } from "./utils.ts";
+import { type RendererProps, type SourceMap } from "./types.ts";
+import {
+    cleanUrl,
+    escape,
+    fixHtmlLocalImageHref,
+    fixLocalImageHref,
+    injectHtmlAttributes,
+    renderSourceMap,
+} from "./utils.ts";
 
 /**
  * An object containing functions to render tokens to HTML.
@@ -12,7 +19,7 @@ export class Renderer {
         this.props = props;
     }
 
-    code(code: string, infostring: string | undefined): string {
+    code(code: string, infostring: string | undefined, sourceMap: SourceMap): string {
         const lang = (infostring ?? "").match(/^\S*/)?.[0];
 
         code = code.replace(/\n$/, "") + "\n";
@@ -21,7 +28,7 @@ export class Renderer {
         const highlightedCode = hljs.highlight(code, { language }).value;
 
         return (
-            '<pre><code class="hljs language-' +
+            `<pre${renderSourceMap(sourceMap)}><code class="hljs language-` +
             escape(language) +
             '">' +
             highlightedCode +
@@ -29,27 +36,36 @@ export class Renderer {
         );
     }
 
-    blockquote(quote: string): string {
-        return `<blockquote>\n${quote}</blockquote>\n`;
+    blockquote(quote: string, sourceMap: SourceMap): string {
+        return `<blockquote${renderSourceMap(sourceMap)}>\n${quote}</blockquote>\n`;
     }
 
-    html(html: string, _block?: boolean): string {
-        return fixHtmlLocalImageHref(html, this.props.localImageUrlPrefix);
+    html(html: string, _block: boolean, sourceMap?: SourceMap | undefined): string {
+        let result = fixHtmlLocalImageHref(html, this.props.localImageUrlPrefix);
+        if (sourceMap) {
+            result = injectHtmlAttributes(result, [
+                ["line-start", sourceMap[0]],
+                ["line-end", sourceMap[1]],
+            ]);
+        }
+        return result;
     }
 
-    heading(text: string, level: number): string {
+    heading(text: string, level: number, sourceMap: SourceMap): string {
         // ignore IDs
-        return `<h${level}>${text}</h${level}>\n`;
+        return `<h${level}${renderSourceMap(sourceMap)}>${text}</h${level}>\n`;
     }
 
-    hr(): string {
-        return "<hr>\n";
+    hr(sourceMap: SourceMap): string {
+        return `<hr${renderSourceMap(sourceMap)}>\n`;
     }
 
-    list(body: string, ordered: boolean, start: number | ""): string {
+    list(body: string, ordered: boolean, start: number | "", sourceMap: SourceMap): string {
         const type = ordered ? "ol" : "ul";
         const startatt = ordered && start !== 1 ? ' start="' + start + '"' : "";
-        return "<" + type + startatt + ">\n" + body + "</" + type + ">\n";
+        return (
+            "<" + type + startatt + `${renderSourceMap(sourceMap)}>\n` + body + "</" + type + ">\n"
+        );
     }
 
     listitem(text: string, _task: boolean, _checked: boolean): string {
@@ -60,18 +76,27 @@ export class Renderer {
         return "<input " + (checked ? 'checked="" ' : "") + 'disabled="" type="checkbox">';
     }
 
-    paragraph(text: string): string {
-        return `<p>${text}</p>\n`;
+    paragraph(text: string, sourceMap: SourceMap): string {
+        return `<p${renderSourceMap(sourceMap)}>${text}</p>\n`;
     }
 
-    table(header: string, body: string): string {
+    table(header: string, body: string, sourceMap: SourceMap): string {
         if (body) body = `<tbody>${body}</tbody>`;
 
-        return "<table>\n" + "<thead>\n" + header + "</thead>\n" + body + "</table>\n";
+        return (
+            `<table${renderSourceMap(sourceMap)}>\n` +
+            "<thead>\n" +
+            header +
+            "</thead>\n" +
+            body +
+            "</table>\n"
+        );
     }
 
-    tablerow(content: string): string {
-        return `<tr>\n${content}</tr>\n`;
+    tablerow(content: string, sourceMapStart: number | undefined): string {
+        return `<tr${renderSourceMap(
+            sourceMapStart ? [sourceMapStart, sourceMapStart] : undefined,
+        )}>\n${content}</tr>\n`;
     }
 
     tablecell(

package/src/sample.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "bun:test";
 
-test("2 + 2", () => {
+test("1 + 3", () => {
     expect(1 + 3).toBe(4);
 });

package/src/tokenizer.ts

@@ -24,6 +24,8 @@ export class Tokenizer {
     space(src: string): Tokens["Space"] | undefined {
         const cap = block.newline.exec(src);
         if (cap && cap[0].length > 0) {
+            // call getSourceMap to increment "this.lexer.line"
+            this.lexer.getSourceMap(cap[0]);
             return {
                 type: "space",
                 raw: cap[0],
@@ -42,6 +44,7 @@ export class Tokenizer {
             raw: cap[0],
             codeBlockStyle: "indented",
             text: rtrim(text, "\n"),
+            sourceMap: this.lexer.getSourceMap(cap[0]),
         };
     }
 
@@ -57,6 +60,7 @@ export class Tokenizer {
             raw,
             lang: cap[2] ? cap[2].trim().replace(inline.escapes, "$1") : cap[2],
             text,
+            sourceMap: this.lexer.getSourceMap(raw),
         };
     }
 
@@ -80,6 +84,7 @@ export class Tokenizer {
             depth: cap[1]!.length,
             text,
             tokens: this.lexer.inline(text),
+            sourceMap: this.lexer.getSourceMap(cap[0]),
         };
     }
 
@@ -90,6 +95,7 @@ export class Tokenizer {
         return {
             type: "hr",
             raw: cap[0],
+            sourceMap: this.lexer.getSourceMap(cap[0]),
         };
     }
 
@@ -97,7 +103,7 @@ export class Tokenizer {
         const cap = block.blockquote.exec(src);
         if (!cap) return undefined;
 
-        const text = cap[0].replace(/^ *>[ \t]?/gm, "");
+        const text = rtrim(cap[0].replace(/^ *>[ \t]?/gm, ""), "\n");
         const top = this.lexer.state.top;
         this.lexer.state.top = true;
         const tokens = this.lexer.blockTokens(text, []);
@@ -107,6 +113,7 @@ export class Tokenizer {
             raw: cap[0],
             tokens,
             text,
+            sourceMap: this.lexer.getSourceMap(cap[0]),
         };
     }
 
@@ -124,6 +131,7 @@ export class Tokenizer {
             start: isordered ? +bull.slice(0, -1) : "",
             loose: false,
             items: [] as Tokens["ListItem"][],
+            sourceMap: [0, 0], // updated below once token.raw is built
         };
 
         bull = isordered ? `\\d{1,9}\\${bull.slice(-1)}` : `\\${bull}`;
@@ -285,6 +293,8 @@ export class Tokenizer {
         list.items[list.items.length - 1]!.text = itemContents.trimEnd();
         list.raw = list.raw.trimEnd();
 
+        list.sourceMap = this.lexer.getSourceMap(list.raw);
+
         // Item child tokens handled here at end because we needed to have the final item to trim it first
         for (let i = 0, listItemsLen = list.items.length; i < listItemsLen; i++) {
             this.lexer.state.top = false;
@@ -321,6 +331,7 @@ export class Tokenizer {
             raw: cap[0],
             pre: cap[1] === "pre" || cap[1] === "script" || cap[1] === "style",
             text: cap[0],
+            sourceMap: this.lexer.getSourceMap(cap[0]),
         };
         return token;
     }
@@ -340,6 +351,7 @@ export class Tokenizer {
             raw: cap[0],
             href,
             title,
+            sourceMap: this.lexer.getSourceMap(cap[0]),
         };
     }
 
@@ -363,6 +375,7 @@ export class Tokenizer {
             })),
             align: [],
             rows: [],
+            sourceMap: this.lexer.getSourceMap(cap[0]),
         };
 
         const align = cap[2].replace(/^\||\| *$/g, "").split("|") as (string | null)[];
@@ -429,6 +442,7 @@ export class Tokenizer {
             depth: cap[2]!.startsWith("=") ? 1 : 2,
             text: cap[1]!,
             tokens: this.lexer.inline(cap[1]!),
+            sourceMap: this.lexer.getSourceMap(cap[0]),
         };
     }
 
@@ -442,6 +456,7 @@ export class Tokenizer {
             raw: cap[0],
             text,
             tokens: this.lexer.inline(text),
+            sourceMap: this.lexer.getSourceMap(cap[0]),
         };
     }
 
@@ -454,6 +469,7 @@ export class Tokenizer {
             raw: cap[0],
             text: cap[0],
             tokens: this.lexer.inline(cap[0]),
+            sourceMap: this.lexer.getSourceMap(cap[0]),
         };
     }
 

package/src/types.ts

@@ -2,8 +2,11 @@ type BaseToken = {
     type: string;
     raw: string;
     text?: string;
+    sourceMap?: SourceMap;
 };
 
+export type SourceMap = [start: number, end: number] | undefined;
+
 // eslint-disable-next-line @typescript-eslint/consistent-type-definitions
 export interface Tokens extends Record<string, BaseToken> {
     Space: {
@@ -16,6 +19,7 @@ export interface Tokens extends Record<string, BaseToken> {
         text: string;
         codeBlockStyle?: "indented";
         lang?: string | undefined;
+        sourceMap: SourceMap;
     };
     Heading: {
         type: "heading";
@@ -23,6 +27,7 @@ export interface Tokens extends Record<string, BaseToken> {
         text: string;
         depth: number;
         tokens: Token[];
+        sourceMap: SourceMap;
     };
     Table: {
         type: "table";
@@ -30,6 +35,7 @@ export interface Tokens extends Record<string, BaseToken> {
         align: ("center" | "left" | "right" | null)[];
         header: Tokens["TableCell"][];
         rows: Tokens["TableCell"][][];
+        sourceMap: SourceMap;
     };
     TableCell: {
         type: "tablecell";
@@ -40,12 +46,14 @@ export interface Tokens extends Record<string, BaseToken> {
     Hr: {
         type: "hr";
         raw: string;
+        sourceMap: SourceMap;
     };
     Blockquote: {
         type: "blockquote";
         raw: string;
         text: string;
         tokens: Token[];
+        sourceMap: SourceMap;
     };
     List: {
         type: "list";
@@ -54,6 +62,7 @@ export interface Tokens extends Record<string, BaseToken> {
         start: number | "";
         loose: boolean;
         items: Tokens["ListItem"][];
+        sourceMap: SourceMap;
     };
     ListItem: {
         type: "list_item";
@@ -70,6 +79,7 @@ export interface Tokens extends Record<string, BaseToken> {
         text: string;
         pre?: boolean | undefined;
         tokens: Token[];
+        sourceMap: SourceMap;
     };
     HTML: {
         type: "html";
@@ -77,12 +87,14 @@ export interface Tokens extends Record<string, BaseToken> {
         text: string;
         pre: boolean;
         block: boolean;
+        sourceMap: SourceMap;
     };
     Text: {
         type: "text";
         raw: string;
         text: string;
         tokens?: Token[];
+        sourceMap?: SourceMap;
     };
     Def: {
         type: "def";
@@ -90,6 +102,7 @@ export interface Tokens extends Record<string, BaseToken> {
         tag: string;
         href: string;
         title: string;
+        sourceMap: SourceMap;
     };
     Escape: {
         type: "escape";

package/src/utils.ts

@@ -1,5 +1,5 @@
 import { type Lexer } from "./lexer.ts";
-import { type Tokens } from "./types.ts";
+import { type SourceMap, type Tokens } from "./types.ts";
 
 /**
  * Helpers
@@ -31,6 +31,27 @@ export function escape(html: string, encode?: boolean) {
     return html;
 }
 
+export function injectHtmlAttributes(
+    html: string,
+    attrs: [name: string, value: string | number][],
+) {
+    const parser = new DOMParser();
+    try {
+        const doc = parser.parseFromString(html, "text/html");
+        // eslint-disable-next-line
+        if (!doc.body) throw Error("Invalid HTML");
+        const element = doc.body.firstChild as HTMLElement;
+        // eslint-disable-next-line
+        if (!element) throw Error("No valid element found");
+        attrs.forEach(([name, value]) => {
+            element.setAttribute(name, JSON.stringify(value));
+        });
+        return element.outerHTML;
+    } catch (_) {
+        return html;
+    }
+}
+
 export function fixHtmlLocalImageHref(html: string, localImageUrlPrefix: string): string {
     return html.replace(
         /<img\s+([^>]*?)src\s*=\s*(["'])([^\2>]+?)\2([^>]*)>/gm,
@@ -65,6 +86,11 @@ export function fixLocalImageHref(href: string, localImageUrlPrefix: string): st
     }
 }
 
+export function renderSourceMap(sourceMap: SourceMap) {
+    if (!sourceMap) return "";
+    return ` line-start=${sourceMap[0]} line-end=${sourceMap[1]}`;
+}
+
 export function cleanUrl(href: string) {
     try {
         href = encodeURI(href).replace(/%25/g, "%");