markit-ai 0.2.0 → 0.3.0

package/dist/converters/pdf/headers.js

@@ -0,0 +1,108 @@
+/**
+ * Running header/footer detection and removal.
+ *
+ * Many PDFs have repeated text at the top or bottom of every page:
+ * document titles, chapter names, page numbers, copyright notices.
+ * These pollute the markdown output as false headings or noise.
+ *
+ * Algorithm:
+ *   1. For each page, bucket text boxes by Y position (top/bottom zones)
+ *   2. Collect the text content at each zone across all pages
+ *   3. Text appearing on >20% of pages OR 8+ consecutive pages is a
+ *      running header/footer
+ *   4. Remove matching text boxes before further processing
+ */
+/** Minimum number of pages to enable header/footer detection. */
+const MIN_PAGES = 5;
+/** Minimum Y position for top zone (from bottom of page in PDF coords). */
+const TOP_ZONE_MIN_Y = 700;
+/** Maximum Y position for bottom zone. */
+const BOTTOM_ZONE_MAX_Y = 80;
+/**
+ * Minimum consecutive pages a text must appear on to be considered a
+ * running header/footer. Catches both document-wide headers (appearing
+ * on every page) and chapter-specific headers (appearing on 4+ consecutive
+ * pages within a chapter).
+ */
+const MIN_CONSECUTIVE_PAGES = 8;
+/**
+ * Detect and remove running headers and footers from all pages.
+ * Mutates the pages array in place, removing header/footer text boxes.
+ *
+ * Uses two strategies:
+ *   1. Global frequency: text appearing on > 20% of all pages
+ *   2. Consecutive runs: text appearing on 8+ consecutive pages
+ */
+export function stripHeadersFooters(pages) {
+    if (pages.length < MIN_PAGES)
+        return;
+    // Step 1: Build per-page zone text sets
+    const pageZoneTexts = [];
+    for (const page of pages) {
+        const zoneTexts = new Set();
+        for (const tb of page.textBoxes) {
+            const midY = (tb.bounds.top + tb.bounds.bottom) / 2;
+            if (midY >= TOP_ZONE_MIN_Y || midY <= BOTTOM_ZONE_MAX_Y) {
+                const key = tb.text.trim().replace(/\s+/g, " ");
+                if (key.length > 0)
+                    zoneTexts.add(key);
+            }
+        }
+        pageZoneTexts.push(zoneTexts);
+    }
+    // Step 2: Count global frequency AND longest consecutive run for each text
+    const globalCount = new Map();
+    const maxConsecutive = new Map();
+    // Collect all unique zone texts
+    const allTexts = new Set();
+    for (const zts of pageZoneTexts) {
+        for (const t of zts)
+            allTexts.add(t);
+    }
+    for (const text of allTexts) {
+        let total = 0;
+        let consecutive = 0;
+        let maxRun = 0;
+        for (const zts of pageZoneTexts) {
+            if (zts.has(text)) {
+                total++;
+                consecutive++;
+                if (consecutive > maxRun)
+                    maxRun = consecutive;
+            }
+            else {
+                consecutive = 0;
+            }
+        }
+        globalCount.set(text, total);
+        maxConsecutive.set(text, maxRun);
+    }
+    // Step 3: Identify running headers/footers
+    const globalThreshold = Math.max(3, Math.floor(pages.length * 0.2));
+    const repeatedTexts = new Set();
+    for (const text of allTexts) {
+        const gc = globalCount.get(text) ?? 0;
+        const mc = maxConsecutive.get(text) ?? 0;
+        // Global: appears on 20%+ of pages
+        if (gc >= globalThreshold) {
+            repeatedTexts.add(text);
+            continue;
+        }
+        // Consecutive: appears on 8+ consecutive pages (chapter-level headers)
+        if (mc >= MIN_CONSECUTIVE_PAGES) {
+            repeatedTexts.add(text);
+        }
+    }
+    if (repeatedTexts.size === 0)
+        return;
+    // Step 4: Remove matching text boxes from each page
+    for (const page of pages) {
+        page.textBoxes = page.textBoxes.filter((tb) => {
+            const midY = (tb.bounds.top + tb.bounds.bottom) / 2;
+            if (midY < TOP_ZONE_MIN_Y && midY > BOTTOM_ZONE_MAX_Y)
+                return true;
+            const normalized = tb.text.trim().replace(/\s+/g, " ");
+            return !repeatedTexts.has(normalized);
+        });
+    }
+}

package/dist/converters/pdf/index.d.ts

@@ -0,0 +1,19 @@
+/**
+ * PDF to Markdown converter.
+ *
+ * Uses mupdf (native WASM) for fast PDF parsing and a custom pipeline for
+ * table detection via vector line extraction + raycasting.
+ *
+ * Pipeline:
+ *   1. Extract text boxes + vector segments + image regions per page (mupdf)
+ *   2. Detect column layout (single vs multi-column)
+ *   3. Per column: detect table grids from segments (grid detection + raycasting)
+ *   4. Render diagrams as PNG files (if output directory provided)
+ *   5. Render tables as markdown tables, free text as paragraphs/headings
+ */
+import type { ConversionResult, Converter, StreamInfo } from "../../types.js";
+export declare class PdfConverter implements Converter {
+    name: string;
+    accepts(streamInfo: StreamInfo): boolean;
+    convert(input: Buffer, streamInfo: StreamInfo): Promise<ConversionResult>;
+}

package/dist/converters/pdf/index.js

@@ -0,0 +1,116 @@
+/**
+ * PDF to Markdown converter.
+ *
+ * Uses mupdf (native WASM) for fast PDF parsing and a custom pipeline for
+ * table detection via vector line extraction + raycasting.
+ *
+ * Pipeline:
+ *   1. Extract text boxes + vector segments + image regions per page (mupdf)
+ *   2. Detect column layout (single vs multi-column)
+ *   3. Per column: detect table grids from segments (grid detection + raycasting)
+ *   4. Render diagrams as PNG files (if output directory provided)
+ *   5. Render tables as markdown tables, free text as paragraphs/headings
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { detectColumns } from "./columns.js";
+import { extractPages, renderImageRegion } from "./extract.js";
+import { resolveTableGrids } from "./grid.js";
+import { stripHeadersFooters } from "./headers.js";
+import { renderPageContent } from "./render.js";
+const EXTENSIONS = [".pdf"];
+const MIMETYPES = ["application/pdf", "application/x-pdf"];
+/**
+ * Process a set of text boxes (one column or full page): run table detection,
+ * separate free text, and render to markdown.
+ */
+function processColumn(pageNumber, textBoxes, segments, imageBlocks) {
+    const { grids, consumedIds } = resolveTableGrids(pageNumber, textBoxes, segments);
+    const consumedSet = new Set(consumedIds);
+    const freeTextBoxes = textBoxes.filter((tb) => !consumedSet.has(tb.id));
+    return renderPageContent(freeTextBoxes, grids, imageBlocks, textBoxes);
+}
+export class PdfConverter {
+    name = "pdf";
+    accepts(streamInfo) {
+        if (streamInfo.extension && EXTENSIONS.includes(streamInfo.extension)) {
+            return true;
+        }
+        if (streamInfo.mimetype &&
+            MIMETYPES.some((m) => streamInfo.mimetype?.startsWith(m))) {
+            return true;
+        }
+        return false;
+    }
+    async convert(input, streamInfo) {
+        const pdfBytes = new Uint8Array(input);
+        const pages = await extractPages(pdfBytes);
+        // Remove running headers/footers before processing
+        stripHeadersFooters(pages);
+        const imageDir = streamInfo.imageDir;
+        if (imageDir) {
+            mkdirSync(imageDir, { recursive: true });
+        }
+        const pageMarkdowns = [];
+        for (const page of pages) {
+            // Build image blocks for this page
+            const imageBlocks = [];
+            if (imageDir && page.images.length > 0) {
+                for (const img of page.images) {
+                    const filename = `${img.id}.png`;
+                    const filepath = join(imageDir, filename);
+                    try {
+                        const png = renderImageRegion(pdfBytes, img);
+                        writeFileSync(filepath, png);
+                        imageBlocks.push({
+                            topY: img.topY,
+                            markdown: `![${img.id}](${filepath})`,
+                        });
+                    }
+                    catch {
+                        // Image rendering failed — skip
+                    }
+                }
+            }
+            else if (page.images.length > 0) {
+                for (const img of page.images) {
+                    imageBlocks.push({
+                        topY: img.topY,
+                        markdown: `<!-- image: ${img.id} (page ${img.pageNumber}, ${img.bbox.w}x${img.bbox.h}pt) -->`,
+                    });
+                }
+            }
+            // Detect column layout
+            const layout = detectColumns(page.textBoxes);
+            if (layout.columnCount === 1) {
+                // Single column — process normally
+                const md = processColumn(page.pageNumber, page.textBoxes, page.segments, imageBlocks);
+                if (md.length > 0)
+                    pageMarkdowns.push(md);
+            }
+            else {
+                // Multi-column — process each column independently, then join
+                const columnMarkdowns = [];
+                for (const colBoxes of layout.columns) {
+                    // Filter segments to those within this column's X range
+                    const colXMin = Math.min(...colBoxes.map((tb) => tb.bounds.left));
+                    const colXMax = Math.max(...colBoxes.map((tb) => tb.bounds.right));
+                    const margin = 10;
+                    const colSegments = page.segments.filter((seg) => {
+                        const segXMin = Math.min(seg.x1, seg.x2);
+                        const segXMax = Math.max(seg.x1, seg.x2);
+                        return segXMax >= colXMin - margin && segXMin <= colXMax + margin;
+                    });
+                    // Images go with the first column only (no X info to split by)
+                    const md = processColumn(page.pageNumber, colBoxes, colSegments, columnMarkdowns.length === 0 ? imageBlocks : []);
+                    if (md.length > 0)
+                        columnMarkdowns.push(md);
+                }
+                const joined = columnMarkdowns.join("\n\n");
+                if (joined.length > 0)
+                    pageMarkdowns.push(joined);
+            }
+        }
+        return { markdown: pageMarkdowns.join("\n\n") };
+    }
+}

package/dist/converters/pdf/render.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Markdown rendering for PDF pages.
+ *
+ * Converts table grids and free text boxes into markdown, handling:
+ * - Table grid → markdown table (`| col | col |`)
+ * - Free text → paragraphs with heading detection (by font size)
+ * - Content ordering (top-to-bottom via Y coordinate)
+ * - Paragraph wrap merging (lines broken across PDF line boundaries)
+ * - Page number removal
+ *
+ * Ported from @oharato/pdf2md-ts, stripped of CJK/TDnet-specific logic.
+ */
+import type { TableGrid, TextBox } from "./types.js";
+/**
+ * Render a TableGrid as a markdown table.
+ */
+export declare function renderTableToMarkdown(table: TableGrid): string;
+/**
+ * Render one page's content: free text and tables interleaved top-to-bottom.
+ */
+export declare function renderPageContent(freeTextBoxes: TextBox[], tables: TableGrid[], imageBlocks?: Array<{
+    topY: number;
+    markdown: string;
+}>, allTextBoxes?: TextBox[]): string;