pantsdown 1.4.1 → 1.4.3

package/README.md

@@ -2,26 +2,42 @@
 
 # Pantsdown
 
+<img src="https://raw.githubusercontent.com/wallpants/pantsdown/main/docs/github.svg" height="60px" align="right" />
+<img src="https://raw.githubusercontent.com/wallpants/pantsdown/main/docs/markdown.svg" height="60px" align="right" />
+
 Pantsdown is a **Markdown** to **"GitHub HTML"** converter based on [Marked](https://github.com/markedjs/marked).
 Basically this is a non customizable version of Marked, optimized to render markdown similar to how GitHub does it plus
 some features developed specifically for [github-preview.nvim](https://github.com/wallpants/github-preview.nvim).
 
-Applicable fixes/updates released by Marked will eventually be applied to Pantsdown.
-
-If you need a feature that's supported by GitHub and is not already listed in Pantsdown's [Roadmap](#roadmap),
+If you need a feature that's supported by GitHub and is not already listed in the [Roadmap](#roadmap),
 feel free to open an issue. If you need anything else, use Marked or another library.
 
+### [Demo](https://wallpants.github.io/pantsdown/)
+
 ## Installation
 
 🚨 Pantsdown is distributed only as a TypeScript module. This means you'll need a bundler to handle transpilation.
 See below for usage examples.
 
 ```sh
+# bun
 bun install pantsdown
+# npm
+npm install pantsdown
 ```
 
 ## Usage
 
+### Styles
+
+For styles to be properly applied, either the element containing the generated html or one of its parents
+must have the classes `class="pantsdown light"` or `class="pantsdown dark"` added.
+
+### [Bun](https://bun.sh/)
+
+Take a look at [how Pantsdown's demo is built](https://github.com/wallpants/pantsdown/blob/main/docs/build.ts)
+for a very simple usage example with Bun.
+
 ### [Vite](https://vitejs.dev/guide/#scaffolding-your-first-vite-project)
 
 Create a Vite Project & install dependencies:
@@ -64,6 +80,8 @@ function App() {
         if (container) container.innerHTML = html;
     }, []);
 
+    // 🚨 for styles to be applied, a parent element must have
+    // the classes "pantsdown light" or "pantsdown dark" added
     return <div id="markdown-container" className="pantsdown light" />;
 }
 
@@ -87,24 +105,35 @@ const config: PantsdownConfig = {
          * relativeImageUrlPrefix: "/__localimage__/"
          *
          * ![image](./wallpants-512.png)
-         * relative src is updated and result in:
+         * relative src is updated and results in:
          * <img src="/__localimage__/wallpants-512.png" />
          *
          * ![image](https://avatars.githubusercontent.com/wallpants)
-         * absolute path remains unchanged:
+         * absolute src remains unchanged:
          * <img src="https://avatars.githubusercontent.com/wallpants" />
          */
         relativeImageUrlPrefix: "/__localimage__/",
+
+        /**
+         * Whether to render <details> html tags with attribute `open=""`
+         *
+         * @default
+         * false
+         */
+        detailsTagDefaultOpen: true,
     },
 };
 
 const pantsdown = new Pantsdown(config);
-pantsdown.parse(markdown);
+const html = pantsdown.parse(markdown);
+
+console.log(html);
 ```
 
 ## Roadmap
 
 -   [x] Add id & anchors to headings
+-   [ ] Add sourcemap to list items
 -   [ ] Add button to copy code in fences
 -   [ ] [Mermaid](https://mermaid.js.org/) support
 -   [ ] [Footnote](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#footnotes) support [Base](https://github.com/bent10/marked-extensions/tree/main/packages/footnote)

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "pantsdown",
-  "version": "1.4.1",
+  "version": "1.4.3",
   "author": "wallpants",
-  "description": "The markdown parser",
+  "description": "Markdown to 'GitHub HTML' parser",
   "license": "MIT",
   "type": "module",
   "repository": {
@@ -29,7 +29,8 @@
   "scripts": {
     "format": "prettier **/*{ts,js} --write",
     "lint": "tsc && eslint . --ext ts --report-unused-disable-directives --max-warnings 0",
-    "commit": "cz"
+    "commit": "cz",
+    "docs:build": "bun run docs/build.ts"
   },
   "release": {
     "branches": [

package/src/lexer.ts

@@ -5,10 +5,10 @@ import { type Links, type SourceMap, type Token } from "./types.ts";
 export class Lexer {
     private tokenizer: Tokenizer;
     private inlineQueue: { src: string; tokens: Token[] }[];
-    private tokens: Token[] = [];
     private links: Links = {};
-    private line = 0;
 
+    tokens: Token[] = [];
+    line = 1;
     state = {
         inLink: false,
         inRawBlock: false,
@@ -25,9 +25,10 @@ export class Lexer {
      */
     lex(src: string) {
         // reset values from previous parse
+        this.tokenizer.pendingHtmlClose = [];
         this.tokens = [];
         this.links = {};
-        this.line = 0;
+        this.line = 1;
 
         src = src.replace(/\r\n|\r/g, "\n");
 

package/src/parser.ts

@@ -76,7 +76,7 @@ export class Parser {
                 }
                 case "blockquote": {
                     const body = this.parse(token.tokens);
-                    out += this.renderer.blockquote(body, token.sourceMap);
+                    out += this.renderer.blockquote(body);
                     continue;
                 }
                 case "list": {
@@ -122,7 +122,12 @@ export class Parser {
                         }
 
                         itemBody += this.parse(item.tokens, loose);
-                        body += this.renderer.listitem(itemBody, task, Boolean(checked));
+                        body += this.renderer.listitem(
+                            itemBody,
+                            task,
+                            Boolean(checked),
+                            token.sourceMap![0] + j,
+                        );
                     }
 
                     const listClasses: string[] = [];

package/src/renderer.ts

@@ -1,5 +1,6 @@
 import GithubSlugger from "github-slugger";
 import hljs from "highlight.js";
+import { inline } from "./rules/inline.ts";
 import { type PantsdownConfig, type SourceMap } from "./types.ts";
 import {
     cleanUrl,
@@ -14,6 +15,7 @@ import {
 
 const defaultConfig: NonNullable<PantsdownConfig["renderer"]> = {
     relativeImageUrlPrefix: "",
+    detailsTagDefaultOpen: false,
 };
 
 /**
@@ -45,19 +47,27 @@ export class Renderer {
         );
     }
 
-    blockquote(quote: string, sourceMap: SourceMap): string {
-        return `<blockquote${renderSourceMap(sourceMap)}>\n${quote}</blockquote>\n`;
+    blockquote(quote: string): string {
+        return `<blockquote>\n${quote}</blockquote>\n`;
     }
 
     html(html: string, _block: boolean, sourceMap?: SourceMap | undefined): string {
-        let result = fixHtmlLocalImageHref(html, this.rendererConfig.relativeImageUrlPrefix);
+        const result = fixHtmlLocalImageHref(html, this.rendererConfig.relativeImageUrlPrefix);
+
+        const htmlAttributes: [name: string, value: string | number][] = [];
+
         if (sourceMap) {
-            result = injectHtmlAttributes(result, [
-                ["line-start", sourceMap[0]],
-                ["line-end", sourceMap[1]],
-            ]);
+            htmlAttributes.push(["line-start", sourceMap[0]], ["line-end", sourceMap[1]]);
         }
-        return result;
+
+        if (this.rendererConfig.detailsTagDefaultOpen) {
+            const tag = inline.tag.exec(html);
+            if (tag?.[0] === "<details>") {
+                htmlAttributes.push(["open", ""]);
+            }
+        }
+
+        return injectHtmlAttributes(result, htmlAttributes);
     }
 
     heading(text: string, level: number, sourceMap: SourceMap): string {
@@ -96,10 +106,12 @@ export class Renderer {
         );
     }
 
-    listitem(text: string, task: boolean, _checked: boolean): string {
+    listitem(text: string, task: boolean, _checked: boolean, lineStart: number): string {
         const classes: string[] = [];
         if (task) classes.push("task-list-item");
-        return `<li${renderHtmlClasses(classes)}>${text}</li>\n`;
+        return `<li${renderHtmlClasses(
+            classes,
+        )} line-start="${lineStart}" line-end="${lineStart}">${text}</li>\n`;
     }
 
     checkbox(checked: boolean, classes: string[] = []): string {

package/src/tokenizer.ts

@@ -16,6 +16,7 @@ import {
  */
 export class Tokenizer {
     private lexer: Lexer;
+    pendingHtmlClose: [tag: string, index: number][] = [];
 
     constructor(lexer: Lexer) {
         this.lexer = lexer;
@@ -108,12 +109,12 @@ export class Tokenizer {
         this.lexer.state.top = true;
         const tokens = this.lexer.blockTokens(text, []);
         this.lexer.state.top = top;
+        this.lexer.line++;
         return {
             type: "blockquote",
             raw: cap[0],
             tokens,
             text,
-            sourceMap: this.lexer.getSourceMap(cap[0]),
         };
     }
 
@@ -333,6 +334,65 @@ export class Tokenizer {
             text: cap[0],
             sourceMap: this.lexer.getSourceMap(cap[0]),
         };
+
+        /*
+         * Sometimes an html token does not contain its closing tag.
+         *
+         * The following markdown:
+         *
+         * 130  <details>
+         * 131    <summary><h4>Hello World</h4></summary>
+         * 132
+         * 133    Here some non-html markdown
+         * 134  </details>
+         *
+         * would result in 3 tokens:
+         * [
+         *   {
+         *     type: "html",
+         *     raw: "<details>↵  <summary><h4>PUT</h4></summary>↵↵",
+         *     sourceMap: [130, 131]
+         *   },
+         *   {
+         *     type: "paragraph",
+         *     raw: "Here some non-html markdown",
+         *     sourceMap: [133, 133]
+         *   },
+         *   {
+         *     type: "html",
+         *     raw: "</details>",
+         *   }
+         * ]
+         *
+         * sourceMap metadata in first token will be incorrect, because it will
+         * not take its children into consideration when it should.
+         *
+         * To solve that we keep track of html tokens that are pending to be closed
+         * and update their sourceMap once they're closed.
+         */
+
+        const capEndsWith = (str: string) => cap[0].trimEnd().endsWith(str);
+
+        const tag = inline.tag.exec(src);
+        const isHtmlClosed = capEndsWith(tag![0].slice(1));
+
+        if (!isHtmlClosed) {
+            // index where the token we just created will be inserted
+            const tokenIdx = this.lexer.tokens.length;
+            // first in last out
+            this.pendingHtmlClose.unshift([tag![0], tokenIdx]);
+        } else if (this.pendingHtmlClose.length) {
+            for (const [pendingTag, index] of this.pendingHtmlClose) {
+                if (capEndsWith(pendingTag.slice(1))) {
+                    const updateToken = this.lexer.tokens[index] as Tokens["HTML"];
+                    if (updateToken.sourceMap?.[1] && token.sourceMap?.[1]) {
+                        updateToken.sourceMap[1] = token.sourceMap[1];
+                    }
+                    this.pendingHtmlClose.shift();
+                }
+            }
+        }
+
         return token;
     }
 

package/src/types.ts

@@ -53,7 +53,6 @@ export interface Tokens extends Record<string, BaseToken> {
         raw: string;
         text: string;
         tokens: Token[];
-        sourceMap: SourceMap;
     };
     List: {
         type: "list";
@@ -197,13 +196,21 @@ export type PantsdownConfig = {
          * relativeImageUrlPrefix: "/__localimage__/"
          *
          * ![image](./wallpants-512.png)
-         * relative src is updated and result in:
+         * relative src is updated and results in:
          * <img src="/__localimage__/wallpants-512.png" />
          *
          * ![image](https://avatars.githubusercontent.com/wallpants)
-         * absolute path remains unchanged:
+         * absolute src remains unchanged:
          * <img src="https://avatars.githubusercontent.com/wallpants" />
          */
-        relativeImageUrlPrefix: string;
+        relativeImageUrlPrefix?: string;
+
+        /**
+         * Whether to render <details> html tags with attribute `open=""`
+         *
+         * @default
+         * false
+         */
+        detailsTagDefaultOpen?: boolean;
     };
 };

package/src/utils.ts

@@ -61,32 +61,50 @@ export function getHtmlElementText(html: string) {
     }
 }
 
+/**
+ * Only works when html element has no other attributes already defined.
+ * For the regex to work the element must be clean of attributes:
+ *
+ * <div>{whatever}</div>
+ */
 export function injectHtmlAttributes(
     html: string,
     attrs: [name: string, value: string | number][],
 ) {
-    try {
-        const element = parseHtmlElement(html);
-        attrs.forEach(([name, value]) => {
-            element.setAttribute(name, JSON.stringify(value));
-        });
-        return element.outerHTML;
-    } catch (_) {
-        return html;
+    if (!attrs.length) return html;
+
+    const closingBracket = /[a-zA-Z0-9]>|\/>/;
+    const match = html.match(closingBracket);
+    if (match) {
+        let htmlAttrs = "";
+        attrs.forEach((atrr) => (htmlAttrs += ` ${atrr[0]}=${atrr[1]}`));
+        const sliceIdx = match.index! + (match[0] === "/>" ? -1 : 1);
+        return html.slice(0, sliceIdx) + htmlAttrs + html.slice(sliceIdx);
     }
+    return html;
 }
 
-export function fixHtmlLocalImageHref(html: string, relativeImageUrlPrefix: string): string {
-    return html.replace(
-        /<img\s+([^>]*?)src\s*=\s*(["'])([^\2>]+?)\2([^>]*)>/gm,
-        (_m, g1, _g2, g3: string, g4) => {
-            const href = fixLocalImageHref(g3, relativeImageUrlPrefix);
-            return `<img ${g1}src="${href}"${g4}>`;
-        },
-    );
+export function fixHtmlLocalImageHref(
+    html: string,
+    relativeImageUrlPrefix: string | undefined,
+): string {
+    return relativeImageUrlPrefix
+        ? html.replace(
+              /<img\s+([^>]*?)src\s*=\s*(["'])([^\2>]+?)\2([^>]*)>/gm,
+              (_m, g1, _g2, g3: string, g4) => {
+                  const href = fixLocalImageHref(g3, relativeImageUrlPrefix);
+                  return `<img ${g1}src="${href}"${g4}>`;
+              },
+          )
+        : html;
 }
 
-export function fixLocalImageHref(href: string, relativeImageUrlPrefix: string): string {
+export function fixLocalImageHref(
+    href: string,
+    relativeImageUrlPrefix: string | undefined,
+): string {
+    if (!relativeImageUrlPrefix) return href;
+
     const reIsAbsolute = /^[\w+]+:\/\//;
     const dummyUrl = "http://__dummy__";
     const dummyBaseUrl = new URL(relativeImageUrlPrefix, dummyUrl);

package/.eslintrc.json

@@ -1,26 +0,0 @@
-{
-  "root": true,
-  "parser": "@typescript-eslint/parser",
-  "parserOptions": {
-    "ecmaVersion": "latest",
-    "sourceType": "module",
-    "project": ["tsconfig.json"]
-  },
-  "extends": [
-    "eslint:recommended",
-    "plugin:@typescript-eslint/strict-type-checked",
-    "plugin:@typescript-eslint/stylistic-type-checked"
-  ],
-  "rules": {
-    "@typescript-eslint/no-non-null-assertion": "off",
-    "@typescript-eslint/consistent-type-definitions": ["error", "type"],
-    "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }],
-    "@typescript-eslint/consistent-type-imports": [
-      "warn",
-      { "prefer": "type-imports", "fixStyle": "inline-type-imports" }
-    ],
-    "no-useless-escape": "off",
-    "no-control-regex": "off",
-    "no-cond-assign": "off"
-  }
-}

package/.github/workflows/release.yaml

@@ -1,45 +0,0 @@
-name: Release
-on:
-    pull_request:
-    push:
-        branches:
-            - main
-
-permissions:
-    contents: read
-
-jobs:
-    Test:
-        runs-on: ubuntu-latest
-        steps:
-            - name: Checkout repo
-              uses: actions/checkout@v4
-            - name: Install bun
-              uses: oven-sh/setup-bun@v1
-            - name: Install dependencies
-              run: bun install
-            - name: Lint
-              run: bun run lint
-            - name: Test
-              run: bun test
-
-    Release:
-        permissions:
-            contents: write
-        needs: [Test]
-        if: |
-            github.ref == 'refs/heads/main' &&
-            github.event.repository.fork == false
-        runs-on: ubuntu-latest
-        steps:
-            - name: Checkout Code
-              uses: actions/checkout@v4
-            - name: Install bun
-              uses: oven-sh/setup-bun@v1
-            - name: Install dependencies
-              run: bun install
-            - name: Release
-              env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-                  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-              run: bun run semantic-release

package/.prettierrc.json

@@ -1,11 +0,0 @@
-{
-  "tabWidth": 4,
-  "printWidth": 100,
-  "plugins": ["prettier-plugin-organize-imports"],
-  "overrides": [
-    {
-      "files": ["*.json"],
-      "options": { "tabWidth": 2 }
-    }
-  ]
-}