markit-ai 0.1.1 → 0.2.0

package/dist/providers/openai.js

@@ -20,7 +20,12 @@ export const openai = {
                                 role: "user",
                                 content: [
                                     { type: "text", text: prompt },
-                                    { type: "image_url", image_url: { url: `data:${mimetype};base64,${image.toString("base64")}` } },
+                                    {
+                                        type: "image_url",
+                                        image_url: {
+                                            url: `data:${mimetype};base64,${image.toString("base64")}`,
+                                        },
+                                    },
                                 ],
                             },
                         ],
@@ -36,7 +41,9 @@ export const openai = {
             },
             transcribe: async (audio, mimetype) => {
                 const ext = mimeToExt(mimetype);
-                const file = new File([audio], `audio${ext}`, { type: mimetype });
+                const file = new File([new Uint8Array(audio)], `audio${ext}`, {
+                    type: mimetype,
+                });
                 const formData = new FormData();
                 formData.append("model", config.transcriptionModel || "gpt-4o-mini-transcribe");
                 formData.append("file", file);
@@ -57,8 +64,12 @@ export const openai = {
 };
 function mimeToExt(mime) {
     const map = {
-        "audio/mpeg": ".mp3", "audio/wav": ".wav", "audio/mp4": ".m4a",
-        "video/mp4": ".mp4", "audio/ogg": ".ogg", "audio/flac": ".flac",
+        "audio/mpeg": ".mp3",
+        "audio/wav": ".wav",
+        "audio/mp4": ".m4a",
+        "video/mp4": ".mp4",
+        "audio/ogg": ".ogg",
+        "audio/flac": ".flac",
         "audio/aac": ".aac",
     };
     return map[mime] || ".mp3";

package/dist/utils/turndown.d.ts

@@ -0,0 +1,8 @@
+import TurndownService from "turndown";
+export declare function createTurndown(): TurndownService;
+/**
+ * Normalize HTML tables so turndown-plugin-gfm can handle them:
+ * - Wrap first row in <thead> if missing
+ * - Strip <p> tags inside <td>/<th> cells
+ */
+export declare function normalizeTablesHtml(html: string): string;

package/dist/utils/turndown.js

@@ -0,0 +1,64 @@
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+export function createTurndown() {
+    const turndown = new TurndownService({
+        headingStyle: "atx",
+        codeBlockStyle: "fenced",
+        bulletListMarker: "-",
+    });
+    turndown.use(gfm);
+    // Fix strikethrough: GFM spec uses ~~ (double tilde), not ~ (single)
+    turndown.addRule("strikethrough", {
+        filter: ["del", "s", "strike"],
+        replacement(content) {
+            return `~~${content}~~`;
+        },
+    });
+    // Fix heading escaping: turndown escapes "1." to "1\." to avoid ordered lists
+    turndown.addRule("heading", {
+        filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
+        replacement(content, node) {
+            const level = Number(node.nodeName.charAt(1));
+            const prefix = "#".repeat(level);
+            // Unescape unnecessary backslash before periods in headings
+            const cleaned = content.replace(/\\([.])/g, "$1").trim();
+            return `\n\n${prefix} ${cleaned}\n\n`;
+        },
+    });
+    // Override listItem rule to use single space after marker (turndown hardcodes 3)
+    turndown.addRule("listItem", {
+        filter: "li",
+        replacement(content, node, options) {
+            content = content
+                .replace(/^\n+/, "")
+                .replace(/\n+$/, "\n")
+                .replace(/\n/gm, "\n  ");
+            const parent = node.parentNode;
+            let prefix = `${options.bulletListMarker} `;
+            if (parent?.nodeName === "OL") {
+                const start = parent.getAttribute("start");
+                const index = Array.prototype.indexOf.call(parent.children, node);
+                prefix = `${(start ? Number(start) : 1) + index}. `;
+            }
+            return prefix + content + (node.nextSibling ? "\n" : "");
+        },
+    });
+    return turndown;
+}
+/**
+ * Normalize HTML tables so turndown-plugin-gfm can handle them:
+ * - Wrap first row in <thead> if missing
+ * - Strip <p> tags inside <td>/<th> cells
+ */
+export function normalizeTablesHtml(html) {
+    // Strip <p> tags inside table cells
+    let result = html.replace(/<(td|th)([^>]*)>\s*<p>([\s\S]*?)<\/p>\s*<\/(td|th)>/gi, "<$1$2>$3</$4>");
+    // Add thead to tables that lack it
+    result = result.replace(/<table([^>]*)>\s*(?:<tbody>\s*)?(<tr[\s\S]*?<\/tr>)([\s\S]*?)<\/(?:tbody>\s*<\/)?table>/gi, (_match, attrs, firstRow, rest) => {
+        const theadRow = firstRow
+            .replace(/<td/gi, "<th")
+            .replace(/<\/td>/gi, "</th>");
+        return `<table${attrs}><thead>${theadRow}</thead><tbody>${rest}</tbody></table>`;
+    });
+    return result;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "markit-ai",
-  "version": "0.1.1",
-  "description": "Convert anything to markdown. PDFs, DOCX, HTML, URLs — everything gets milled.",
+  "version": "0.2.0",
+  "description": "Convert anything to markdown. PDF, DOCX, PPTX, XLSX, HTML, EPUB, Jupyter, RSS, images, audio, URLs, and more. Pluggable converters, built-in LLM providers for image description and audio transcription. Works as a CLI and as a library.",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -54,6 +54,7 @@
   "devDependencies": {
     "@biomejs/biome": "^2.3.14",
     "@types/jszip": "^3.4.1",
+    "@types/node": "^25.5.0",
     "@types/turndown": "^5.0.5",
     "typescript": "^5.8.0"
   },
@@ -67,6 +68,7 @@
     "music-metadata": "^11.12.3",
     "rss-parser": "^3.13.0",
     "turndown": "^7.2.0",
+    "turndown-plugin-gfm": "^1.0.2",
     "unpdf": "^1.4.0"
   }
 }

package/dist/llm.d.ts

@@ -1,10 +0,0 @@
-import type { MarkitOptions } from "./types.js";
-import type { MarkitConfig } from "./config.js";
-/**
- * Build describe/transcribe functions from .markit/config.json + env vars.
- * Uses the OpenAI-compatible provider by default.
- */
-export declare function createLlmFunctions(config: MarkitConfig): MarkitOptions;
-export declare function createOpenAIDescribe(config: MarkitConfig): (image: Buffer, mimetype: string) => Promise<string>;
-export declare function createOpenAITranscribe(config: MarkitConfig): (audio: Buffer, mimetype: string) => Promise<string>;
-export declare function createAnthropicDescribe(config: MarkitConfig): (image: Buffer, mimetype: string) => Promise<string>;

package/dist/llm.js

@@ -1,139 +0,0 @@
-import { resolveApiKey, resolveApiBase, resolveModel, resolveTranscriptionModel, } from "./config.js";
-/**
- * Build describe/transcribe functions from .markit/config.json + env vars.
- * Uses the OpenAI-compatible provider by default.
- */
-export function createLlmFunctions(config) {
-    const apiKey = resolveApiKey(config);
-    if (!apiKey)
-        return {};
-    const provider = config.llm?.provider || "openai";
-    if (provider === "anthropic") {
-        return {
-            describe: createAnthropicDescribe(config),
-            // Anthropic doesn't have a transcription API — leave undefined
-        };
-    }
-    return {
-        describe: createOpenAIDescribe(config),
-        transcribe: createOpenAITranscribe(config),
-    };
-}
-// ── OpenAI-compatible (also works with Groq, Together, Fireworks, Ollama) ──
-export function createOpenAIDescribe(config) {
-    const apiKey = resolveApiKey(config);
-    const baseUrl = resolveApiBase(config).replace(/\/+$/, "");
-    const model = resolveModel(config);
-    return async (image, mimetype) => {
-        const base64 = image.toString("base64");
-        const res = await fetch(`${baseUrl}/chat/completions`, {
-            method: "POST",
-            headers: {
-                "Content-Type": "application/json",
-                Authorization: `Bearer ${apiKey}`,
-            },
-            body: JSON.stringify({
-                model,
-                messages: [
-                    {
-                        role: "user",
-                        content: [
-                            { type: "text", text: "Write a detailed description of this image." },
-                            { type: "image_url", image_url: { url: `data:${mimetype};base64,${base64}` } },
-                        ],
-                    },
-                ],
-                max_tokens: 1024,
-            }),
-        });
-        if (!res.ok) {
-            const body = await res.text();
-            throw new Error(`OpenAI API error ${res.status}: ${body}`);
-        }
-        const data = await res.json();
-        return data.choices?.[0]?.message?.content ?? "";
-    };
-}
-export function createOpenAITranscribe(config) {
-    const apiKey = resolveApiKey(config);
-    const baseUrl = resolveApiBase(config).replace(/\/+$/, "");
-    const transcriptionModel = resolveTranscriptionModel(config);
-    return async (audio, mimetype) => {
-        const ext = mimeToExt(mimetype);
-        const file = new File([audio], `audio${ext}`, { type: mimetype });
-        const formData = new FormData();
-        formData.append("model", transcriptionModel);
-        formData.append("file", file);
-        const res = await fetch(`${baseUrl}/audio/transcriptions`, {
-            method: "POST",
-            headers: { Authorization: `Bearer ${apiKey}` },
-            body: formData,
-        });
-        if (!res.ok) {
-            const body = await res.text();
-            throw new Error(`Transcription API error ${res.status}: ${body}`);
-        }
-        const data = await res.json();
-        return data.text ?? "";
-    };
-}
-// ── Anthropic ───────────────────────────────────────────────────────────────
-export function createAnthropicDescribe(config) {
-    const apiKey = process.env.ANTHROPIC_API_KEY ||
-        config.llm?.apiKey ||
-        resolveApiKey(config);
-    const baseUrl = (process.env.ANTHROPIC_BASE_URL ||
-        config.llm?.apiBase ||
-        "https://api.anthropic.com").replace(/\/+$/, "");
-    const model = resolveModel(config, undefined) || "claude-sonnet-4-20250514";
-    return async (image, mimetype) => {
-        const base64 = image.toString("base64");
-        const res = await fetch(`${baseUrl}/v1/messages`, {
-            method: "POST",
-            headers: {
-                "Content-Type": "application/json",
-                "x-api-key": apiKey,
-                "anthropic-version": "2023-06-01",
-            },
-            body: JSON.stringify({
-                model,
-                max_tokens: 1024,
-                messages: [
-                    {
-                        role: "user",
-                        content: [
-                            {
-                                type: "image",
-                                source: {
-                                    type: "base64",
-                                    media_type: mimetype,
-                                    data: base64,
-                                },
-                            },
-                            { type: "text", text: "Write a detailed description of this image." },
-                        ],
-                    },
-                ],
-            }),
-        });
-        if (!res.ok) {
-            const body = await res.text();
-            throw new Error(`Anthropic API error ${res.status}: ${body}`);
-        }
-        const data = await res.json();
-        return data.content?.[0]?.text ?? "";
-    };
-}
-// ── Helpers ─────────────────────────────────────────────────────────────────
-function mimeToExt(mime) {
-    const map = {
-        "audio/mpeg": ".mp3",
-        "audio/wav": ".wav",
-        "audio/mp4": ".m4a",
-        "video/mp4": ".mp4",
-        "audio/ogg": ".ogg",
-        "audio/flac": ".flac",
-        "audio/aac": ".aac",
-    };
-    return map[mime] || ".mp3";
-}

package/dist/mill.d.ts

@@ -1,18 +0,0 @@
-import type { ConversionResult, StreamInfo, ConvertOptions } from "./types.js";
-export declare class Mill {
-    private converters;
-    private options;
-    constructor(options?: ConvertOptions);
-    /**
-     * Convert a local file to markdown.
-     */
-    convertFile(path: string): Promise<ConversionResult>;
-    /**
-     * Convert a URL to markdown.
-     */
-    convertUrl(url: string): Promise<ConversionResult>;
-    /**
-     * Convert a buffer with stream info to markdown.
-     */
-    convert(input: Buffer, streamInfo: StreamInfo): Promise<ConversionResult>;
-}

package/dist/mill.js

@@ -1,123 +0,0 @@
-import { readFileSync } from "node:fs";
-import { extname, basename } from "node:path";
-import { PdfConverter } from "./converters/pdf.js";
-import { DocxConverter } from "./converters/docx.js";
-import { PptxConverter } from "./converters/pptx.js";
-import { XlsxConverter } from "./converters/xlsx.js";
-import { EpubConverter } from "./converters/epub.js";
-import { IpynbConverter } from "./converters/ipynb.js";
-import { HtmlConverter } from "./converters/html.js";
-import { WikipediaConverter } from "./converters/wikipedia.js";
-import { RssConverter } from "./converters/rss.js";
-import { CsvConverter } from "./converters/csv.js";
-import { JsonConverter } from "./converters/json.js";
-import { YamlConverter } from "./converters/yaml.js";
-import { XmlConverter } from "./converters/xml.js";
-import { ZipConverter } from "./converters/zip.js";
-import { ImageConverter } from "./converters/image.js";
-import { AudioConverter } from "./converters/audio.js";
-import { PlainTextConverter } from "./converters/plain-text.js";
-export class Mill {
-    converters = [];
-    options;
-    constructor(options = {}) {
-        this.options = options;
-        // Order matters: specific formats first, generic last.
-        // URL-specific converters (Wikipedia) before generic HTML.
-        // ZIP converter gets a reference to other converters for recursive conversion.
-        const specific = [
-            new PdfConverter(),
-            new DocxConverter(),
-            new PptxConverter(),
-            new XlsxConverter(),
-            new EpubConverter(),
-            new IpynbConverter(),
-            new WikipediaConverter(),
-            new RssConverter(),
-            new CsvConverter(),
-            new JsonConverter(),
-            new YamlConverter(),
-            new ImageConverter(),
-            new AudioConverter(),
-        ];
-        const generic = [
-            new XmlConverter(),
-            new HtmlConverter(),
-        ];
-        // ZIP gets all other converters for recursive extraction
-        const allNonZip = [...specific, ...generic];
-        const zipConverter = new ZipConverter(allNonZip);
-        // Plain text is the ultimate catch-all
-        this.converters = [
-            ...specific,
-            zipConverter,
-            ...generic,
-            new PlainTextConverter(),
-        ];
-    }
-    /**
-     * Convert a local file to markdown.
-     */
-    async convertFile(path) {
-        const buffer = readFileSync(path);
-        const streamInfo = {
-            localPath: path,
-            extension: extname(path).toLowerCase(),
-            filename: basename(path),
-        };
-        return this.convert(buffer, streamInfo);
-    }
-    /**
-     * Convert a URL to markdown.
-     */
-    async convertUrl(url) {
-        const response = await fetch(url, {
-            headers: {
-                Accept: "text/markdown, text/html;q=0.9, text/plain;q=0.8, */*;q=0.1",
-                "User-Agent": "mill/0.1.0",
-            },
-        });
-        if (!response.ok) {
-            throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
-        }
-        const contentType = response.headers.get("content-type") || "";
-        const [mimetype] = contentType.split(";");
-        // Derive extension from URL path
-        const urlPath = new URL(url).pathname;
-        const ext = extname(urlPath).toLowerCase();
-        const buffer = Buffer.from(await response.arrayBuffer());
-        const streamInfo = {
-            url,
-            mimetype: mimetype.trim(),
-            extension: ext || undefined,
-            filename: basename(urlPath) || undefined,
-        };
-        return this.convert(buffer, streamInfo);
-    }
-    /**
-     * Convert a buffer with stream info to markdown.
-     */
-    async convert(input, streamInfo) {
-        const errors = [];
-        for (const converter of this.converters) {
-            if (!converter.accepts(streamInfo))
-                continue;
-            try {
-                return await converter.convert(input, streamInfo, this.options);
-            }
-            catch (err) {
-                errors.push({
-                    converter: converter.name,
-                    error: err instanceof Error ? err : new Error(String(err)),
-                });
-            }
-        }
-        if (errors.length > 0) {
-            const details = errors
-                .map((e) => `  ${e.converter}: ${e.error.message}`)
-                .join("\n");
-            throw new Error(`Conversion failed:\n${details}`);
-        }
-        throw new Error(`Unsupported format: ${streamInfo.extension || streamInfo.mimetype || "unknown"}`);
-    }
-}