markit-ai 0.2.0 → 0.4.0

package/dist/converters/pptx.js

@@ -1,3 +1,5 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
 import { XMLParser } from "fast-xml-parser";
 import JSZip from "jszip";
 const EXTENSIONS = [".pptx"];
@@ -14,12 +16,13 @@ export class PptxConverter {
             return true;
         return false;
     }
-    async convert(input, _streamInfo) {
+    async convert(input, streamInfo) {
         const zip = await JSZip.loadAsync(input);
         const parser = new XMLParser({
             ignoreAttributes: false,
             attributeNamePrefix: "@_",
             textNodeName: "#text",
+            processEntities: { maxTotalExpansions: 1_000_000 },
         });
         // Get slide order from presentation.xml
         const presXml = await zip.file("ppt/presentation.xml")?.async("string");
@@ -66,7 +69,12 @@ export class PptxConverter {
             });
             slidePaths.push(...slideFiles);
         }
+        const imageDir = streamInfo.imageDir;
+        if (imageDir) {
+            mkdirSync(imageDir, { recursive: true });
+        }
         const sections = [];
+        let imageCount = 0;
         for (let i = 0; i < slidePaths.length; i++) {
             const slideXml = await zip.file(slidePaths[i])?.async("string");
             if (!slideXml)
@@ -75,6 +83,17 @@ export class PptxConverter {
             const spTree = slide["p:sld"]?.["p:cSld"]?.["p:spTree"];
             if (!spTree)
                 continue;
+            // Parse slide-level rels for image references
+            const slideRelsPath = `${slidePaths[i].replace("slides/slide", "slides/_rels/slide")}.rels`;
+            const slideRelsXml = await zip.file(slideRelsPath)?.async("string");
+            const slideRelMap = new Map();
+            if (slideRelsXml) {
+                const slideRels = parser.parse(slideRelsXml);
+                const relItems = toList(slideRels?.Relationships?.Relationship);
+                for (const r of relItems) {
+                    slideRelMap.set(r["@_Id"], r["@_Target"]);
+                }
+            }
             const slideLines = [`<!-- Slide ${i + 1} -->`];
             const shapes = spTree["p:sp"];
             const shapeList = Array.isArray(shapes) ? shapes : shapes ? [shapes] : [];
@@ -91,6 +110,55 @@ export class PptxConverter {
                     slideLines.push(text);
                 }
             }
+            // Extract embedded images
+            const pics = toList(spTree["p:pic"]);
+            for (const pic of pics) {
+                const blipFill = pic["p:blipFill"];
+                const rEmbed = blipFill?.["a:blip"]?.["@_r:embed"];
+                if (!rEmbed)
+                    continue;
+                const target = slideRelMap.get(rEmbed);
+                if (!target)
+                    continue;
+                // Resolve relative target against slide directory
+                const imagePath = target.startsWith("/")
+                    ? target.slice(1)
+                    : `ppt/slides/${target}`;
+                // Normalize path (e.g. ppt/slides/../media/image1.png → ppt/media/image1.png)
+                const normalizedPath = imagePath
+                    .split("/")
+                    .reduce((parts, seg) => {
+                    if (seg === "..")
+                        parts.pop();
+                    else
+                        parts.push(seg);
+                    return parts;
+                }, [])
+                    .join("/");
+                const imageFile = zip.file(normalizedPath);
+                if (!imageFile)
+                    continue;
+                imageCount++;
+                const name = pic["p:nvSpPr"]?.["p:cNvPr"]?.["@_name"] ||
+                    pic["p:nvPicPr"]?.["p:cNvPr"]?.["@_name"] ||
+                    `image_${imageCount}`;
+                if (imageDir) {
+                    try {
+                        const ext = normalizedPath.split(".").pop() || "png";
+                        const filename = `slide${i + 1}_${imageCount}.${ext}`;
+                        const filepath = join(imageDir, filename);
+                        const buf = await imageFile.async("nodebuffer");
+                        writeFileSync(filepath, buf);
+                        slideLines.push(`![${name}](${filepath})`);
+                    }
+                    catch {
+                        slideLines.push(`<!-- image: ${name} (slide ${i + 1}) -->`);
+                    }
+                }
+                else {
+                    slideLines.push(`<!-- image: ${name} (slide ${i + 1}) -->`);
+                }
+            }
             // Tables
             const graphicFrames = spTree["p:graphicFrame"];
             const gfList = Array.isArray(graphicFrames)
@@ -214,3 +282,8 @@ export class PptxConverter {
         return lines.join("\n");
     }
 }
+function toList(val) {
+    if (!val)
+        return [];
+    return Array.isArray(val) ? val : [val];
+}

package/dist/converters/xlsx.js

@@ -20,6 +20,7 @@ export class XlsxConverter {
             ignoreAttributes: false,
             attributeNamePrefix: "@_",
             textNodeName: "#text",
+            processEntities: { maxTotalExpansions: 1_000_000 },
         });
         // Parse shared strings
         const ssXml = await zip.file("xl/sharedStrings.xml")?.async("string");

package/dist/index.d.ts

@@ -3,11 +3,12 @@ export { AudioConverter } from "./converters/audio.js";
 export { CsvConverter } from "./converters/csv.js";
 export { DocxConverter } from "./converters/docx.js";
 export { EpubConverter } from "./converters/epub.js";
+export { GitHubConverter } from "./converters/github.js";
 export { HtmlConverter } from "./converters/html.js";
 export { ImageConverter } from "./converters/image.js";
 export { IpynbConverter } from "./converters/ipynb.js";
 export { JsonConverter } from "./converters/json.js";
-export { PdfConverter } from "./converters/pdf.js";
+export { PdfConverter } from "./converters/pdf/index.js";
 export { PlainTextConverter } from "./converters/plain-text.js";
 export { PptxConverter } from "./converters/pptx.js";
 export { RssConverter } from "./converters/rss.js";

package/dist/index.js

@@ -2,11 +2,12 @@ export { AudioConverter } from "./converters/audio.js";
 export { CsvConverter } from "./converters/csv.js";
 export { DocxConverter } from "./converters/docx.js";
 export { EpubConverter } from "./converters/epub.js";
+export { GitHubConverter } from "./converters/github.js";
 export { HtmlConverter } from "./converters/html.js";
 export { ImageConverter } from "./converters/image.js";
 export { IpynbConverter } from "./converters/ipynb.js";
 export { JsonConverter } from "./converters/json.js";
-export { PdfConverter } from "./converters/pdf.js";
+export { PdfConverter } from "./converters/pdf/index.js";
 export { PlainTextConverter } from "./converters/plain-text.js";
 export { PptxConverter } from "./converters/pptx.js";
 export { RssConverter } from "./converters/rss.js";

package/dist/main.js

@@ -18,6 +18,7 @@ program
     .option("-q, --quiet", "Raw markdown only, no decoration")
     .option("-p, --prompt <text>", "Extra instructions for image description")
     .option("-o, --output <file>", "Write to file instead of stdout")
+    .option("-i, --image-dir <dir>", "Extract images to this directory")
     .addHelpText("after", `
 Examples:
   $ markit report.pdf                  Convert a PDF to markdown
@@ -42,7 +43,8 @@ program
         json: globals.json,
         quiet: globals.quiet,
         output: opts.output,
-        prompt: opts.prompt,
+        prompt: globals.prompt,
+        imageDir: globals.imageDir,
     });
 });
 program
@@ -144,6 +146,7 @@ program.on("command:*", async (args) => {
         quiet: globals.quiet,
         output: globals.output,
         prompt: globals.prompt,
+        imageDir: globals.imageDir,
     });
 });
 // No args → show concise help

package/dist/markit.d.ts

@@ -7,7 +7,7 @@ export declare class Markit {
     /**
      * Convert a local file to markdown.
      */
-    convertFile(path: string): Promise<ConversionResult>;
+    convertFile(path: string, extra?: Partial<StreamInfo>): Promise<ConversionResult>;
     /**
      * Convert a URL to markdown.
      */

package/dist/markit.js

@@ -4,11 +4,12 @@ import { AudioConverter } from "./converters/audio.js";
 import { CsvConverter } from "./converters/csv.js";
 import { DocxConverter } from "./converters/docx.js";
 import { EpubConverter } from "./converters/epub.js";
+import { GitHubConverter } from "./converters/github.js";
 import { HtmlConverter } from "./converters/html.js";
 import { ImageConverter } from "./converters/image.js";
 import { IpynbConverter } from "./converters/ipynb.js";
 import { JsonConverter } from "./converters/json.js";
-import { PdfConverter } from "./converters/pdf.js";
+import { PdfConverter } from "./converters/pdf/index.js";
 import { PlainTextConverter } from "./converters/plain-text.js";
 import { PptxConverter } from "./converters/pptx.js";
 import { RssConverter } from "./converters/rss.js";
@@ -32,6 +33,7 @@ export class Markit {
             new XlsxConverter(),
             new EpubConverter(),
             new IpynbConverter(),
+            new GitHubConverter(),
             new WikipediaConverter(),
             new RssConverter(),
             new CsvConverter(),
@@ -56,12 +58,13 @@ export class Markit {
     /**
      * Convert a local file to markdown.
      */
-    async convertFile(path) {
+    async convertFile(path, extra) {
         const buffer = readFileSync(path);
         const streamInfo = {
             localPath: path,
             extension: extname(path).toLowerCase(),
             filename: basename(path),
+            ...extra,
         };
         return this.convert(buffer, streamInfo);
     }
@@ -69,6 +72,18 @@ export class Markit {
      * Convert a URL to markdown.
      */
     async convertUrl(url) {
+        // Let converters with a URL-specific hook handle it first
+        const streamInfo = { url };
+        for (const converter of this.converters) {
+            if (!converter.convertUrl || !converter.accepts(streamInfo))
+                continue;
+            try {
+                return await converter.convertUrl(url, this.options);
+            }
+            catch {
+                // Fall through to default fetch path
+            }
+        }
         const response = await fetch(url, {
             headers: {
                 Accept: "text/markdown, text/html;q=0.9, text/plain;q=0.8, */*;q=0.1",
@@ -84,13 +99,13 @@ export class Markit {
         const urlPath = new URL(url).pathname;
         const ext = extname(urlPath).toLowerCase();
         const buffer = Buffer.from(await response.arrayBuffer());
-        const streamInfo = {
+        const fetchedInfo = {
             url,
             mimetype: mimetype.trim(),
             extension: ext || undefined,
             filename: basename(urlPath) || undefined,
         };
-        return this.convert(buffer, streamInfo);
+        return this.convert(buffer, fetchedInfo);
     }
     /**
      * Convert a buffer with stream info to markdown.

package/dist/types.d.ts

@@ -5,6 +5,8 @@ export interface StreamInfo {
     filename?: string;
     localPath?: string;
     url?: string;
+    /** Directory to write extracted images/diagrams. */
+    imageDir?: string;
 }
 export interface ConversionResult {
     markdown: string;
@@ -23,6 +25,12 @@ export interface Converter {
     name: string;
     /** Quick check: can this converter handle the given stream? */
     accepts(streamInfo: StreamInfo): boolean;
+    /**
+     * Optional URL-first hook. When present, called before the default fetch
+     * so the converter can handle URL fetching itself (e.g. rewrite to a raw
+     * content URL or call an API).
+     */
+    convertUrl?(url: string, options?: MarkitOptions): Promise<ConversionResult>;
     /** Convert the source to markdown */
     convert(input: Buffer, streamInfo: StreamInfo, options?: MarkitOptions): Promise<ConversionResult>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "markit-ai",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Convert anything to markdown. PDF, DOCX, PPTX, XLSX, HTML, EPUB, Jupyter, RSS, images, audio, URLs, and more. Pluggable converters, built-in LLM providers for image description and audio transcription. Works as a CLI and as a library.",
   "type": "module",
   "main": "dist/index.js",
@@ -65,10 +65,10 @@
     "fast-xml-parser": "^5.5.9",
     "jszip": "^3.10.1",
     "mammoth": "^1.9.0",
+    "mupdf": "^1.27.0",
     "music-metadata": "^11.12.3",
     "rss-parser": "^3.13.0",
     "turndown": "^7.2.0",
-    "turndown-plugin-gfm": "^1.0.2",
-    "unpdf": "^1.4.0"
+    "turndown-plugin-gfm": "^1.0.2"
   }
 }

package/dist/converters/pdf.d.ts

@@ -1,6 +0,0 @@
-import type { ConversionResult, Converter, StreamInfo } from "../types.js";
-export declare class PdfConverter implements Converter {
-    name: string;
-    accepts(streamInfo: StreamInfo): boolean;
-    convert(input: Buffer, _streamInfo: StreamInfo): Promise<ConversionResult>;
-}

package/dist/converters/pdf.js

@@ -1,29 +0,0 @@
-const EXTENSIONS = [".pdf"];
-const MIMETYPES = ["application/pdf", "application/x-pdf"];
-export class PdfConverter {
-    name = "pdf";
-    accepts(streamInfo) {
-        if (streamInfo.extension && EXTENSIONS.includes(streamInfo.extension)) {
-            return true;
-        }
-        if (streamInfo.mimetype &&
-            MIMETYPES.some((m) => streamInfo.mimetype?.startsWith(m))) {
-            return true;
-        }
-        return false;
-    }
-    async convert(input, _streamInfo) {
-        let extractText;
-        try {
-            ({ extractText } = await import("unpdf"));
-        }
-        catch {
-            throw new Error("PDF support requires 'unpdf'. Install it: npm install unpdf");
-        }
-        const result = await extractText(new Uint8Array(input));
-        const text = Array.isArray(result.text)
-            ? result.text.join("\n\n")
-            : String(result.text);
-        return { markdown: text.trim() };
-    }
-}