markit-ai 0.1.3 → 0.3.0

package/dist/utils/turndown.js

@@ -0,0 +1,64 @@
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+export function createTurndown() {
+    const turndown = new TurndownService({
+        headingStyle: "atx",
+        codeBlockStyle: "fenced",
+        bulletListMarker: "-",
+    });
+    turndown.use(gfm);
+    // Fix strikethrough: GFM spec uses ~~ (double tilde), not ~ (single)
+    turndown.addRule("strikethrough", {
+        filter: ["del", "s", "strike"],
+        replacement(content) {
+            return `~~${content}~~`;
+        },
+    });
+    // Fix heading escaping: turndown escapes "1." to "1\." to avoid ordered lists
+    turndown.addRule("heading", {
+        filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
+        replacement(content, node) {
+            const level = Number(node.nodeName.charAt(1));
+            const prefix = "#".repeat(level);
+            // Unescape unnecessary backslash before periods in headings
+            const cleaned = content.replace(/\\([.])/g, "$1").trim();
+            return `\n\n${prefix} ${cleaned}\n\n`;
+        },
+    });
+    // Override listItem rule to use single space after marker (turndown hardcodes 3)
+    turndown.addRule("listItem", {
+        filter: "li",
+        replacement(content, node, options) {
+            content = content
+                .replace(/^\n+/, "")
+                .replace(/\n+$/, "\n")
+                .replace(/\n/gm, "\n  ");
+            const parent = node.parentNode;
+            let prefix = `${options.bulletListMarker} `;
+            if (parent?.nodeName === "OL") {
+                const start = parent.getAttribute("start");
+                const index = Array.prototype.indexOf.call(parent.children, node);
+                prefix = `${(start ? Number(start) : 1) + index}. `;
+            }
+            return prefix + content + (node.nextSibling ? "\n" : "");
+        },
+    });
+    return turndown;
+}
+/**
+ * Normalize HTML tables so turndown-plugin-gfm can handle them:
+ * - Wrap first row in <thead> if missing
+ * - Strip <p> tags inside <td>/<th> cells
+ */
+export function normalizeTablesHtml(html) {
+    // Strip <p> tags inside table cells
+    let result = html.replace(/<(td|th)([^>]*)>\s*<p>([\s\S]*?)<\/p>\s*<\/(td|th)>/gi, "<$1$2>$3</$4>");
+    // Add thead to tables that lack it
+    result = result.replace(/<table([^>]*)>\s*(?:<tbody>\s*)?(<tr[\s\S]*?<\/tr>)([\s\S]*?)<\/(?:tbody>\s*<\/)?table>/gi, (_match, attrs, firstRow, rest) => {
+        const theadRow = firstRow
+            .replace(/<td/gi, "<th")
+            .replace(/<\/td>/gi, "</th>");
+        return `<table${attrs}><thead>${theadRow}</thead><tbody>${rest}</tbody></table>`;
+    });
+    return result;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "markit-ai",
-  "version": "0.1.3",
-  "description": "Convert anything to markdown. PDFs, DOCX, HTML, URLs — everything gets milled.",
+  "version": "0.3.0",
+  "description": "Convert anything to markdown. PDF, DOCX, PPTX, XLSX, HTML, EPUB, Jupyter, RSS, images, audio, URLs, and more. Pluggable converters, built-in LLM providers for image description and audio transcription. Works as a CLI and as a library.",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -65,9 +65,10 @@
     "fast-xml-parser": "^5.5.9",
     "jszip": "^3.10.1",
     "mammoth": "^1.9.0",
+    "mupdf": "^1.27.0",
     "music-metadata": "^11.12.3",
     "rss-parser": "^3.13.0",
     "turndown": "^7.2.0",
-    "unpdf": "^1.4.0"
+    "turndown-plugin-gfm": "^1.0.2"
   }
 }

package/dist/converters/pdf.d.ts

@@ -1,6 +0,0 @@
-import type { ConversionResult, Converter, StreamInfo } from "../types.js";
-export declare class PdfConverter implements Converter {
-    name: string;
-    accepts(streamInfo: StreamInfo): boolean;
-    convert(input: Buffer, _streamInfo: StreamInfo): Promise<ConversionResult>;
-}

package/dist/converters/pdf.js

@@ -1,29 +0,0 @@
-const EXTENSIONS = [".pdf"];
-const MIMETYPES = ["application/pdf", "application/x-pdf"];
-export class PdfConverter {
-    name = "pdf";
-    accepts(streamInfo) {
-        if (streamInfo.extension && EXTENSIONS.includes(streamInfo.extension)) {
-            return true;
-        }
-        if (streamInfo.mimetype &&
-            MIMETYPES.some((m) => streamInfo.mimetype?.startsWith(m))) {
-            return true;
-        }
-        return false;
-    }
-    async convert(input, _streamInfo) {
-        let extractText;
-        try {
-            ({ extractText } = await import("unpdf"));
-        }
-        catch {
-            throw new Error("PDF support requires 'unpdf'. Install it: npm install unpdf");
-        }
-        const result = await extractText(new Uint8Array(input));
-        const text = Array.isArray(result.text)
-            ? result.text.join("\n\n")
-            : String(result.text);
-        return { markdown: text.trim() };
-    }
-}