markit-ai 0.2.0 → 0.4.0

package/dist/commands/convert.d.ts

@@ -2,4 +2,5 @@ import type { OutputOptions } from "../utils/output.js";
 export declare function convert(source: string, options: OutputOptions & {
     output?: string;
     prompt?: string;
+    imageDir?: string;
 }): Promise<void>;

package/dist/commands/convert.js

@@ -1,4 +1,6 @@
-import { writeFileSync } from "node:fs";
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { loadConfig } from "../config.js";
 import { Markit } from "../markit.js";
 import { loadAllPlugins } from "../plugins/loader.js";
@@ -23,6 +25,8 @@ export async function convert(source, options) {
     }
     const llmFunctions = createLlmFunctions(config, options.prompt);
     const markit = new Markit(llmFunctions, plugins);
+    // Auto-create a temp dir for images if not explicitly provided
+    const imageDir = options.imageDir || mkdtempSync(join(tmpdir(), "markit-images-"));
     try {
         let result;
         const isStdin = source === "-";
@@ -36,7 +40,7 @@ export async function convert(source, options) {
                 process.exit(EXIT_ERROR);
             }
             const buffer = await readStdin();
-            result = await markit.convert(buffer, {});
+            result = await markit.convert(buffer, { imageDir });
         }
         else if (isUrl) {
             // Progress hint for URL fetches (stderr so it doesn't pollute piped output)
@@ -46,7 +50,7 @@ export async function convert(source, options) {
             result = await markit.convertUrl(source);
         }
         else {
-            result = await markit.convertFile(source);
+            result = await markit.convertFile(source, { imageDir });
         }
         const label = isStdin ? "stdin" : source;
         // Write to file or stdout

package/dist/commands/formats.js

@@ -23,6 +23,11 @@ const BUILTIN_FORMATS = [
         extensions: [".mp3", ".wav", ".m4a", ".flac"],
         builtin: true,
     },
+    {
+        name: "GitHub",
+        extensions: ["github.com/*", "gist.github.com/*"],
+        builtin: true,
+    },
     { name: "ZIP", extensions: [".zip"], builtin: true },
     {
         name: "Plain text",

package/dist/converters/docx.d.ts

@@ -2,5 +2,5 @@ import type { ConversionResult, Converter, StreamInfo } from "../types.js";
 export declare class DocxConverter implements Converter {
     name: string;
     accepts(streamInfo: StreamInfo): boolean;
-    convert(input: Buffer, _streamInfo: StreamInfo): Promise<ConversionResult>;
+    convert(input: Buffer, streamInfo: StreamInfo): Promise<ConversionResult>;
 }

package/dist/converters/docx.js

@@ -1,3 +1,5 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
 import mammoth from "mammoth";
 import { createTurndown, normalizeTablesHtml } from "../utils/turndown.js";
 const EXTENSIONS = [".docx"];
@@ -16,10 +18,40 @@ export class DocxConverter {
         }
         return false;
     }
-    async convert(input, _streamInfo) {
-        const { value: html } = await mammoth.convertToHtml({ buffer: input });
+    async convert(input, streamInfo) {
+        const imageDir = streamInfo.imageDir;
+        if (imageDir) {
+            mkdirSync(imageDir, { recursive: true });
+        }
+        let imageCount = 0;
+        const convertImage = imageDir
+            ? mammoth.images.imgElement((image) => {
+                imageCount++;
+                const ext = (image.contentType?.split("/")[1] || "png").replace("jpeg", "jpg");
+                const filename = `image_${imageCount}.${ext}`;
+                const filepath = join(imageDir, filename);
+                return image.read("base64").then((base64) => {
+                    writeFileSync(filepath, Buffer.from(base64, "base64"));
+                    return { src: filepath, alt: `image_${imageCount}` };
+                });
+            })
+            : mammoth.images.imgElement((image) => {
+                imageCount++;
+                const contentType = image.contentType || "image/png";
+                return image.read("base64").then((base64) => {
+                    return {
+                        src: `data:${contentType};base64,${base64.slice(0, 0)}`,
+                        alt: `image_${imageCount}`,
+                    };
+                });
+            });
+        const { value: html } = await mammoth.convertToHtml({ buffer: input }, { convertImage });
         const turndown = createTurndown();
-        const markdown = turndown.turndown(normalizeTablesHtml(html));
+        let markdown = turndown.turndown(normalizeTablesHtml(html));
+        // Replace data URI images with comment placeholders when no imageDir
+        if (!imageDir) {
+            markdown = markdown.replace(/!\[([^\]]*)\]\(data:[^)]*\)/g, "<!-- image: $1 -->");
+        }
         return { markdown: markdown.trim() };
     }
 }

package/dist/converters/epub.js

@@ -23,6 +23,7 @@ export class EpubConverter {
             ignoreAttributes: false,
             attributeNamePrefix: "@_",
             textNodeName: "#text",
+            processEntities: { maxTotalExpansions: 1_000_000 },
         });
         // Find content.opf path from container.xml
         const containerXml = await zip

package/dist/converters/github.d.ts

@@ -0,0 +1,18 @@
+import type { ConversionResult, Converter, MarkitOptions, StreamInfo } from "../types.js";
+/**
+ * Matches GitHub URLs and fetches clean markdown content directly
+ * from raw endpoints or the GitHub API — no HTML scraping needed.
+ *
+ * Supported patterns:
+ *   - Repos:   github.com/owner/repo          → raw README.md
+ *   - Files:   github.com/owner/repo/blob/…   → raw file content
+ *   - Gists:   gist.github.com/owner/id       → raw gist content
+ *   - Issues:  github.com/owner/repo/issues/N  → API (title + body)
+ *   - PRs:     github.com/owner/repo/pull/N    → API (title + body)
+ */
+export declare class GitHubConverter implements Converter {
+    name: string;
+    accepts(streamInfo: StreamInfo): boolean;
+    convertUrl(url: string, _options?: MarkitOptions): Promise<ConversionResult>;
+    convert(_input: Buffer, streamInfo: StreamInfo): Promise<ConversionResult>;
+}

package/dist/converters/github.js

@@ -0,0 +1,148 @@
+const GITHUB_HOSTS = new Set([
+    "github.com",
+    "www.github.com",
+    "gist.github.com",
+]);
+/**
+ * Matches GitHub URLs and fetches clean markdown content directly
+ * from raw endpoints or the GitHub API — no HTML scraping needed.
+ *
+ * Supported patterns:
+ *   - Repos:   github.com/owner/repo          → raw README.md
+ *   - Files:   github.com/owner/repo/blob/…   → raw file content
+ *   - Gists:   gist.github.com/owner/id       → raw gist content
+ *   - Issues:  github.com/owner/repo/issues/N  → API (title + body)
+ *   - PRs:     github.com/owner/repo/pull/N    → API (title + body)
+ */
+export class GitHubConverter {
+    name = "github";
+    accepts(streamInfo) {
+        if (!streamInfo.url)
+            return false;
+        try {
+            const { hostname } = new URL(streamInfo.url);
+            return GITHUB_HOSTS.has(hostname);
+        }
+        catch {
+            return false;
+        }
+    }
+    async convertUrl(url, _options) {
+        const parsed = new URL(url);
+        if (parsed.hostname === "gist.github.com") {
+            return fetchGist(parsed);
+        }
+        const segments = parsed.pathname.split("/").filter(Boolean);
+        // Need at least owner/repo
+        if (segments.length < 2) {
+            throw new Error(`Unsupported GitHub URL: ${url}`);
+        }
+        const [owner, repo, type, ...rest] = segments;
+        // github.com/owner/repo/blob/ref/path → raw file
+        if (type === "blob" && rest.length >= 2) {
+            const ref = rest[0];
+            const filePath = rest.slice(1).join("/");
+            return fetchRawFile(owner, repo, ref, filePath);
+        }
+        // github.com/owner/repo/issues/N or /pull/N
+        if ((type === "issues" || type === "pull") && rest[0]) {
+            const number = Number.parseInt(rest[0], 10);
+            if (!Number.isNaN(number)) {
+                return fetchIssueOrPr(owner, repo, number);
+            }
+        }
+        // github.com/owner/repo (no subpath or tree/wiki/etc) → README
+        if (!type) {
+            return fetchReadme(owner, repo);
+        }
+        throw new Error(`Unsupported GitHub URL pattern: ${url}`);
+    }
+    async convert(_input, streamInfo) {
+        // GitHub URLs are handled entirely via convertUrl.
+        // If we end up here, the URL was already fetched by the default path —
+        // just delegate to convertUrl.
+        if (streamInfo.url) {
+            return this.convertUrl(streamInfo.url);
+        }
+        throw new Error("GitHub converter requires a URL");
+    }
+}
+// ---------------------------------------------------------------------------
+// Fetchers
+// ---------------------------------------------------------------------------
+async function fetchReadme(owner, repo) {
+    const url = `https://raw.githubusercontent.com/${owner}/${repo}/HEAD/README.md`;
+    const res = await fetch(url);
+    if (!res.ok) {
+        throw new Error(`Failed to fetch README: ${res.status} ${res.statusText}`);
+    }
+    const markdown = (await res.text()).trim();
+    const title = extractFirstHeading(markdown) ?? `${owner}/${repo}`;
+    return { markdown, title };
+}
+async function fetchRawFile(owner, repo, ref, filePath) {
+    const url = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${filePath}`;
+    const res = await fetch(url);
+    if (!res.ok) {
+        throw new Error(`Failed to fetch file: ${res.status} ${res.statusText}`);
+    }
+    const content = (await res.text()).trim();
+    const filename = filePath.split("/").pop() ?? filePath;
+    // If it's markdown, return as-is. Otherwise wrap in a code block.
+    if (filePath.endsWith(".md") || filePath.endsWith(".mdx")) {
+        const title = extractFirstHeading(content) ?? filename;
+        return { markdown: content, title };
+    }
+    const ext = filename.includes(".") ? filename.split(".").pop() : "";
+    const markdown = `# ${filename}\n\n\`\`\`${ext}\n${content}\n\`\`\``;
+    return { markdown, title: filename };
+}
+async function fetchGist(parsed) {
+    const segments = parsed.pathname.split("/").filter(Boolean);
+    // gist.github.com/owner/id
+    const [owner, id] = segments;
+    if (!owner || !id) {
+        throw new Error(`Unsupported gist URL: ${parsed.href}`);
+    }
+    const url = `https://gist.githubusercontent.com/${owner}/${id}/raw`;
+    const res = await fetch(url);
+    if (!res.ok) {
+        throw new Error(`Failed to fetch gist: ${res.status} ${res.statusText}`);
+    }
+    const content = (await res.text()).trim();
+    const title = `gist:${id}`;
+    return { markdown: content, title };
+}
+async function fetchIssueOrPr(owner, repo, number) {
+    const url = `https://api.github.com/repos/${owner}/${repo}/issues/${number}`;
+    const res = await fetch(url, {
+        headers: { Accept: "application/vnd.github.v3+json" },
+    });
+    if (!res.ok) {
+        throw new Error(`Failed to fetch issue/PR: ${res.status} ${res.statusText}`);
+    }
+    const data = (await res.json());
+    const title = data.title ?? `#${number}`;
+    const parts = [`# ${title}`];
+    // Metadata line
+    const meta = [];
+    if (data.user?.login)
+        meta.push(`@${data.user.login}`);
+    if (data.state)
+        meta.push(data.state);
+    if (data.labels?.length) {
+        meta.push(data.labels.map((l) => l.name).join(", "));
+    }
+    if (meta.length > 0)
+        parts.push(meta.join(" · "));
+    if (data.body?.trim())
+        parts.push(data.body.trim());
+    return { markdown: parts.join("\n\n"), title };
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function extractFirstHeading(markdown) {
+    const match = markdown.match(/^#\s+(.+)$/m);
+    return match?.[1]?.trim();
+}

package/dist/converters/pdf/columns.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Multi-column layout detection and text box reordering.
+ *
+ * Many PDFs (legal documents, datasheets, academic papers) use two-column
+ * layouts. Without column detection, text boxes are ordered by Y position
+ * only, interleaving left and right column content.
+ *
+ * Algorithm:
+ *   1. Collect left edges of all text boxes on the page
+ *   2. Find the largest horizontal gap between consecutive left edges
+ *   3. If gap > MIN_GAP_RATIO of the text width and both sides have
+ *      enough boxes → multi-column detected
+ *   4. Assign each text box to a column based on its center X
+ *   5. Return columns in reading order (left-to-right, top-to-bottom)
+ *
+ * This only detects the column structure. The caller is responsible for
+ * processing each column's text boxes independently (table detection,
+ * rendering, etc.).
+ */
+import type { TextBox } from "./types.js";
+export interface ColumnLayout {
+    /** Number of columns detected (1 = single column, 2+ = multi-column). */
+    columnCount: number;
+    /** Text boxes grouped by column, in reading order (left to right). */
+    columns: TextBox[][];
+    /** X positions of column boundaries (between columns). */
+    boundaries: number[];
+}
+/**
+ * Detect column layout and return text boxes grouped by column.
+ *
+ * For single-column pages, returns all boxes in one group.
+ * For multi-column pages, returns boxes split by column in reading order.
+ */
+export declare function detectColumns(textBoxes: TextBox[]): ColumnLayout;

package/dist/converters/pdf/columns.js

@@ -0,0 +1,93 @@
+/**
+ * Multi-column layout detection and text box reordering.
+ *
+ * Many PDFs (legal documents, datasheets, academic papers) use two-column
+ * layouts. Without column detection, text boxes are ordered by Y position
+ * only, interleaving left and right column content.
+ *
+ * Algorithm:
+ *   1. Collect left edges of all text boxes on the page
+ *   2. Find the largest horizontal gap between consecutive left edges
+ *   3. If gap > MIN_GAP_RATIO of the text width and both sides have
+ *      enough boxes → multi-column detected
+ *   4. Assign each text box to a column based on its center X
+ *   5. Return columns in reading order (left-to-right, top-to-bottom)
+ *
+ * This only detects the column structure. The caller is responsible for
+ * processing each column's text boxes independently (table detection,
+ * rendering, etc.).
+ */
+/**
+ * Minimum gap as a fraction of the total text width to consider a column
+ * boundary. A two-column layout typically has ~50% gap; we use a lower
+ * threshold to catch asymmetric columns.
+ */
+const MIN_GAP_RATIO = 0.15;
+/** Minimum number of text boxes on each side of the gap. */
+const MIN_BOXES_PER_COLUMN = 4;
+/** Minimum gap in absolute points to avoid splitting on small whitespace. */
+const MIN_GAP_PTS = 40;
+/**
+ * Detect column layout and return text boxes grouped by column.
+ *
+ * For single-column pages, returns all boxes in one group.
+ * For multi-column pages, returns boxes split by column in reading order.
+ */
+export function detectColumns(textBoxes) {
+    if (textBoxes.length < MIN_BOXES_PER_COLUMN * 2) {
+        return { columnCount: 1, columns: [textBoxes], boundaries: [] };
+    }
+    // Collect unique left edges (rounded to avoid float noise)
+    const lefts = [
+        ...new Set(textBoxes.map((tb) => Math.round(tb.bounds.left))),
+    ].sort((a, b) => a - b);
+    if (lefts.length < 2) {
+        return { columnCount: 1, columns: [textBoxes], boundaries: [] };
+    }
+    const textXMin = lefts[0];
+    const textXMax = Math.max(...textBoxes.map((tb) => Math.round(tb.bounds.right)));
+    const textWidth = textXMax - textXMin;
+    if (textWidth <= 0) {
+        return { columnCount: 1, columns: [textBoxes], boundaries: [] };
+    }
+    // Find the largest gap between consecutive left-edge positions
+    let maxGap = 0;
+    let gapLeft = 0;
+    let gapRight = 0;
+    for (let i = 1; i < lefts.length; i++) {
+        const gap = lefts[i] - lefts[i - 1];
+        if (gap > maxGap) {
+            maxGap = gap;
+            gapLeft = lefts[i - 1];
+            gapRight = lefts[i];
+        }
+    }
+    const gapRatio = maxGap / textWidth;
+    if (gapRatio < MIN_GAP_RATIO || maxGap < MIN_GAP_PTS) {
+        return { columnCount: 1, columns: [textBoxes], boundaries: [] };
+    }
+    // Split point is the midpoint of the gap
+    const splitX = (gapLeft + gapRight) / 2;
+    // Assign boxes to columns based on center X
+    const leftCol = [];
+    const rightCol = [];
+    for (const tb of textBoxes) {
+        const cx = (tb.bounds.left + tb.bounds.right) / 2;
+        if (cx < splitX) {
+            leftCol.push(tb);
+        }
+        else {
+            rightCol.push(tb);
+        }
+    }
+    // Validate both columns have enough content
+    if (leftCol.length < MIN_BOXES_PER_COLUMN ||
+        rightCol.length < MIN_BOXES_PER_COLUMN) {
+        return { columnCount: 1, columns: [textBoxes], boundaries: [] };
+    }
+    return {
+        columnCount: 2,
+        columns: [leftCol, rightCol],
+        boundaries: [splitX],
+    };
+}

package/dist/converters/pdf/extract.d.ts

@@ -0,0 +1,19 @@
+/**
+ * PDF content extraction using mupdf.
+ *
+ * Extracts text boxes (with position, font size, bold) and vector line
+ * segments (table borders) from each page. Uses mupdf's native WASM
+ * engine for fast parsing, and reads raw content streams for vector graphics.
+ *
+ * Coordinate system: PDF native (origin = bottom-left, Y increases upward).
+ */
+import type { ImageRegion, PageContent } from "./types.js";
+/**
+ * Render an image region from a PDF page as a PNG buffer.
+ * Uses mupdf's DrawDevice to render just the cropped area at 2x resolution.
+ */
+export declare function renderImageRegion(input: Uint8Array, region: ImageRegion): Uint8Array;
+/**
+ * Extract text boxes and vector segments from all pages of a PDF buffer.
+ */
+export declare function extractPages(input: Uint8Array): Promise<PageContent[]>;