markit-ai 0.1.3 → 0.2.0

package/dist/converters/docx.js

@@ -1,5 +1,5 @@
 import mammoth from "mammoth";
-import TurndownService from "turndown";
+import { createTurndown, normalizeTablesHtml } from "../utils/turndown.js";
 const EXTENSIONS = [".docx"];
 const MIMETYPES = [
     "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
@@ -18,11 +18,8 @@ export class DocxConverter {
     }
     async convert(input, _streamInfo) {
         const { value: html } = await mammoth.convertToHtml({ buffer: input });
-        const turndown = new TurndownService({
-            headingStyle: "atx",
-            codeBlockStyle: "fenced",
-        });
-        const markdown = turndown.turndown(html);
+        const turndown = createTurndown();
+        const markdown = turndown.turndown(normalizeTablesHtml(html));
         return { markdown: markdown.trim() };
     }
 }

package/dist/converters/epub.js

@@ -1,6 +1,6 @@
 import { XMLParser } from "fast-xml-parser";
 import JSZip from "jszip";
-import TurndownService from "turndown";
+import { createTurndown, normalizeTablesHtml } from "../utils/turndown.js";
 const EXTENSIONS = [".epub"];
 const MIMETYPES = [
     "application/epub",
@@ -75,10 +75,7 @@ export class EpubConverter {
         const basePath = opfPath.includes("/")
             ? opfPath.substring(0, opfPath.lastIndexOf("/"))
             : "";
-        const turndown = new TurndownService({
-            headingStyle: "atx",
-            codeBlockStyle: "fenced",
-        });
+        const turndown = createTurndown();
         const sections = [];
         // Add metadata header
         const metaLines = [];
@@ -101,7 +98,7 @@ export class EpubConverter {
             const cleaned = html
                 .replace(/<script[\s\S]*?<\/script>/gi, "")
                 .replace(/<style[\s\S]*?<\/style>/gi, "");
-            const md = turndown.turndown(cleaned).trim();
+            const md = turndown.turndown(normalizeTablesHtml(cleaned)).trim();
             if (md)
                 sections.push(md);
         }

package/dist/converters/html.js

@@ -1,4 +1,4 @@
-import TurndownService from "turndown";
+import { createTurndown, normalizeTablesHtml } from "../utils/turndown.js";
 const EXTENSIONS = [".html", ".htm"];
 const MIMETYPES = ["text/html", "application/xhtml"];
 export class HtmlConverter {
@@ -16,15 +16,12 @@ export class HtmlConverter {
     async convert(input, streamInfo) {
         const charset = streamInfo.charset || "utf-8";
         const html = new TextDecoder(charset).decode(input);
-        const turndown = new TurndownService({
-            headingStyle: "atx",
-            codeBlockStyle: "fenced",
-        });
+        const turndown = createTurndown();
         // Remove script and style tags before converting
         const cleaned = html
             .replace(/<script[\s\S]*?<\/script>/gi, "")
             .replace(/<style[\s\S]*?<\/style>/gi, "");
-        const markdown = turndown.turndown(cleaned);
+        const markdown = turndown.turndown(normalizeTablesHtml(cleaned));
         // Try to extract title
         const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
         const title = titleMatch ? titleMatch[1].trim() : undefined;

package/dist/converters/rss.js

@@ -1,4 +1,4 @@
-import TurndownService from "turndown";
+import { createTurndown } from "../utils/turndown.js";
 const _EXTENSIONS = [".rss", ".atom", ".xml"];
 const MIMETYPES = [
     "application/rss+xml",
@@ -36,7 +36,7 @@ export class RssConverter {
         throw new Error("Not an RSS or Atom feed");
     }
     parseRss(xml) {
-        const turndown = new TurndownService({ headingStyle: "atx" });
+        const turndown = createTurndown();
         const sections = [];
         // Extract from the <channel> block specifically
         const channelMatch = xml.match(/<channel>([\s\S]*?)<\/channel>/i);
@@ -74,7 +74,7 @@ export class RssConverter {
         return { markdown: sections.join("\n\n").trim(), title: channelTitle };
     }
     parseAtom(xml) {
-        const turndown = new TurndownService({ headingStyle: "atx" });
+        const turndown = createTurndown();
         const sections = [];
         const feedTitle = this.extract(xml, "title");
         const subtitle = this.extract(xml, "subtitle");

package/dist/converters/wikipedia.js

@@ -1,4 +1,4 @@
-import TurndownService from "turndown";
+import { createTurndown } from "../utils/turndown.js";
 const WIKIPEDIA_RE = /^https?:\/\/[a-zA-Z]{2,3}\.wikipedia\.org\//;
 export class WikipediaConverter {
     name = "wikipedia";
@@ -16,10 +16,7 @@ export class WikipediaConverter {
         const title = titleMatch
             ? titleMatch[1].replace(/ - Wikipedia$/, "").trim()
             : undefined;
-        const turndown = new TurndownService({
-            headingStyle: "atx",
-            codeBlockStyle: "fenced",
-        });
+        const turndown = createTurndown();
         // Clean up Wikipedia-specific elements
         let content = contentMatch ? contentMatch[1] : html;
         content = content

package/dist/utils/turndown.d.ts

@@ -0,0 +1,8 @@
+import TurndownService from "turndown";
+export declare function createTurndown(): TurndownService;
+/**
+ * Normalize HTML tables so turndown-plugin-gfm can handle them:
+ * - Wrap first row in <thead> if missing
+ * - Strip <p> tags inside <td>/<th> cells
+ */
+export declare function normalizeTablesHtml(html: string): string;

package/dist/utils/turndown.js

@@ -0,0 +1,64 @@
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+export function createTurndown() {
+    const turndown = new TurndownService({
+        headingStyle: "atx",
+        codeBlockStyle: "fenced",
+        bulletListMarker: "-",
+    });
+    turndown.use(gfm);
+    // Fix strikethrough: GFM spec uses ~~ (double tilde), not ~ (single)
+    turndown.addRule("strikethrough", {
+        filter: ["del", "s", "strike"],
+        replacement(content) {
+            return `~~${content}~~`;
+        },
+    });
+    // Fix heading escaping: turndown escapes "1." to "1\." to avoid ordered lists
+    turndown.addRule("heading", {
+        filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
+        replacement(content, node) {
+            const level = Number(node.nodeName.charAt(1));
+            const prefix = "#".repeat(level);
+            // Unescape unnecessary backslash before periods in headings
+            const cleaned = content.replace(/\\([.])/g, "$1").trim();
+            return `\n\n${prefix} ${cleaned}\n\n`;
+        },
+    });
+    // Override listItem rule to use single space after marker (turndown hardcodes 3)
+    turndown.addRule("listItem", {
+        filter: "li",
+        replacement(content, node, options) {
+            content = content
+                .replace(/^\n+/, "")
+                .replace(/\n+$/, "\n")
+                .replace(/\n/gm, "\n  ");
+            const parent = node.parentNode;
+            let prefix = `${options.bulletListMarker} `;
+            if (parent?.nodeName === "OL") {
+                const start = parent.getAttribute("start");
+                const index = Array.prototype.indexOf.call(parent.children, node);
+                prefix = `${(start ? Number(start) : 1) + index}. `;
+            }
+            return prefix + content + (node.nextSibling ? "\n" : "");
+        },
+    });
+    return turndown;
+}
+/**
+ * Normalize HTML tables so turndown-plugin-gfm can handle them:
+ * - Wrap first row in <thead> if missing
+ * - Strip <p> tags inside <td>/<th> cells
+ */
+export function normalizeTablesHtml(html) {
+    // Strip <p> tags inside table cells
+    let result = html.replace(/<(td|th)([^>]*)>\s*<p>([\s\S]*?)<\/p>\s*<\/(td|th)>/gi, "<$1$2>$3</$4>");
+    // Add thead to tables that lack it
+    result = result.replace(/<table([^>]*)>\s*(?:<tbody>\s*)?(<tr[\s\S]*?<\/tr>)([\s\S]*?)<\/(?:tbody>\s*<\/)?table>/gi, (_match, attrs, firstRow, rest) => {
+        const theadRow = firstRow
+            .replace(/<td/gi, "<th")
+            .replace(/<\/td>/gi, "</th>");
+        return `<table${attrs}><thead>${theadRow}</thead><tbody>${rest}</tbody></table>`;
+    });
+    return result;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "markit-ai",
-  "version": "0.1.3",
-  "description": "Convert anything to markdown. PDFs, DOCX, HTML, URLs — everything gets milled.",
+  "version": "0.2.0",
+  "description": "Convert anything to markdown. PDF, DOCX, PPTX, XLSX, HTML, EPUB, Jupyter, RSS, images, audio, URLs, and more. Pluggable converters, built-in LLM providers for image description and audio transcription. Works as a CLI and as a library.",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -68,6 +68,7 @@
     "music-metadata": "^11.12.3",
     "rss-parser": "^3.13.0",
     "turndown": "^7.2.0",
+    "turndown-plugin-gfm": "^1.0.2",
     "unpdf": "^1.4.0"
   }
 }