@oh-my-pi/pi-coding-agent 16.0.7 → 16.0.8

package/src/markit/converters/pdf/headers.ts

@@ -0,0 +1,106 @@
+// Adapted from markit-ai (MIT). See ../../NOTICE.
+
+/**
+ * Running header/footer detection and removal.
+ *
+ * Many PDFs have repeated text at the top or bottom of every page:
+ * document titles, chapter names, page numbers, copyright notices.
+ * These pollute the markdown output as false headings or noise.
+ *
+ * Algorithm:
+ *   1. For each page, bucket text boxes by Y position (top/bottom zones)
+ *   2. Collect the text content at each zone across all pages
+ *   3. Text appearing on >20% of pages OR 8+ consecutive pages is a
+ *      running header/footer
+ *   4. Remove matching text boxes before further processing
+ */
+import type { PageContent } from "./types";
+
+/** Minimum number of pages to enable header/footer detection. */
+const MIN_PAGES = 5;
+/** Minimum Y position for top zone (from bottom of page in PDF coords). */
+const TOP_ZONE_MIN_Y = 700;
+/** Maximum Y position for bottom zone. */
+const BOTTOM_ZONE_MAX_Y = 80;
+/**
+ * Minimum consecutive pages a text must appear on to be considered a
+ * running header/footer. Catches both document-wide headers (appearing
+ * on every page) and chapter-specific headers (appearing on 4+ consecutive
+ * pages within a chapter).
+ */
+const MIN_CONSECUTIVE_PAGES = 8;
+
+/**
+ * Detect and remove running headers and footers from all pages.
+ * Mutates the pages array in place, removing header/footer text boxes.
+ *
+ * Uses two strategies:
+ *   1. Global frequency: text appearing on > 20% of all pages
+ *   2. Consecutive runs: text appearing on 8+ consecutive pages
+ */
+export function stripHeadersFooters(pages: PageContent[]): void {
+	if (pages.length < MIN_PAGES) return;
+	// Step 1: Build per-page zone text sets
+	const pageZoneTexts: Set<string>[] = [];
+	for (const page of pages) {
+		const zoneTexts = new Set<string>();
+		for (const tb of page.textBoxes) {
+			const midY = (tb.bounds.top + tb.bounds.bottom) / 2;
+			if (midY >= TOP_ZONE_MIN_Y || midY <= BOTTOM_ZONE_MAX_Y) {
+				const key = tb.text.trim().replace(/\s+/g, " ");
+				if (key.length > 0) zoneTexts.add(key);
+			}
+		}
+		pageZoneTexts.push(zoneTexts);
+	}
+	// Step 2: Count global frequency AND longest consecutive run for each text
+	const globalCount = new Map<string, number>();
+	const maxConsecutive = new Map<string, number>();
+	// Collect all unique zone texts
+	const allTexts = new Set<string>();
+	for (const zts of pageZoneTexts) {
+		for (const t of zts) allTexts.add(t);
+	}
+	for (const text of allTexts) {
+		let total = 0;
+		let consecutive = 0;
+		let maxRun = 0;
+		for (const zts of pageZoneTexts) {
+			if (zts.has(text)) {
+				total++;
+				consecutive++;
+				if (consecutive > maxRun) maxRun = consecutive;
+			} else {
+				consecutive = 0;
+			}
+		}
+		globalCount.set(text, total);
+		maxConsecutive.set(text, maxRun);
+	}
+	// Step 3: Identify running headers/footers
+	const globalThreshold = Math.max(3, Math.floor(pages.length * 0.2));
+	const repeatedTexts = new Set<string>();
+	for (const text of allTexts) {
+		const gc = globalCount.get(text) ?? 0;
+		const mc = maxConsecutive.get(text) ?? 0;
+		// Global: appears on 20%+ of pages
+		if (gc >= globalThreshold) {
+			repeatedTexts.add(text);
+			continue;
+		}
+		// Consecutive: appears on 8+ consecutive pages (chapter-level headers)
+		if (mc >= MIN_CONSECUTIVE_PAGES) {
+			repeatedTexts.add(text);
+		}
+	}
+	if (repeatedTexts.size === 0) return;
+	// Step 4: Remove matching text boxes from each page
+	for (const page of pages) {
+		page.textBoxes = page.textBoxes.filter(tb => {
+			const midY = (tb.bounds.top + tb.bounds.bottom) / 2;
+			if (midY < TOP_ZONE_MIN_Y && midY > BOTTOM_ZONE_MAX_Y) return true;
+			const normalized = tb.text.trim().replace(/\s+/g, " ");
+			return !repeatedTexts.has(normalized);
+		});
+	}
+}

package/src/markit/converters/pdf/index.ts

@@ -0,0 +1,146 @@
+// Adapted from markit-ai (MIT). See ../../NOTICE.
+
+/**
+ * PDF to Markdown converter.
+ *
+ * Uses mupdf (native WASM) for fast PDF parsing and a custom pipeline for
+ * table detection via vector line extraction + raycasting.
+ *
+ * Pipeline:
+ *   1. Extract text boxes + vector segments + image regions per page (mupdf)
+ *   2. Detect column layout (single vs multi-column)
+ *   3. Per column: detect table grids from segments (grid detection + raycasting)
+ *   4. Render diagrams as PNG files (if output directory provided)
+ *   5. Render tables as markdown tables, free text as paragraphs/headings
+ */
+import * as path from "node:path";
+import type { ConversionResult, Converter, StreamInfo } from "../../types";
+import { detectColumns } from "./columns";
+import { extractPages, renderImageRegion } from "./extract";
+import { resolveTableGrids } from "./grid";
+import { stripHeadersFooters } from "./headers";
+import { renderPageContent } from "./render";
+import type { Segment, TextBox } from "./types";
+
+const EXTENSIONS = [".pdf"];
+const MIMETYPES = ["application/pdf", "application/x-pdf"];
+
+type ImageBlock = { topY: number; markdown: string };
+
+/**
+ * Process a set of text boxes (one column or full page): run table detection,
+ * separate free text, and render to markdown.
+ */
+function processColumn(
+	pageNumber: number,
+	textBoxes: TextBox[],
+	segments: Segment[],
+	imageBlocks: ImageBlock[],
+): string {
+	const { grids, consumedIds } = resolveTableGrids(pageNumber, textBoxes, segments);
+	const consumedSet = new Set(consumedIds);
+	const freeTextBoxes = textBoxes.filter(tb => !consumedSet.has(tb.id));
+	return renderPageContent(freeTextBoxes, grids, imageBlocks, textBoxes);
+}
+
+export class PdfConverter implements Converter {
+	name = "pdf";
+
+	accepts(streamInfo: StreamInfo): boolean {
+		if (streamInfo.extension && EXTENSIONS.includes(streamInfo.extension)) {
+			return true;
+		}
+		if (streamInfo.mimetype && MIMETYPES.some(m => streamInfo.mimetype?.startsWith(m))) {
+			return true;
+		}
+		return false;
+	}
+
+	async convert(input: Buffer, streamInfo: StreamInfo): Promise<ConversionResult> {
+		const pdfBytes = new Uint8Array(input);
+		const pages = await extractPages(pdfBytes);
+		// Remove running headers/footers before processing.
+		stripHeadersFooters(pages);
+		const imageDir = streamInfo.imageDir;
+
+		const pageMarkdowns: string[] = [];
+		for (const page of pages) {
+			// Build image blocks for this page.
+			const imageBlocks: ImageBlock[] = [];
+			if (imageDir && page.images.length > 0) {
+				for (const img of page.images) {
+					const filename = `${img.id}.png`;
+					const filepath = path.join(imageDir, filename);
+					try {
+						const png = await renderImageRegion(pdfBytes, img);
+						await Bun.write(filepath, png);
+						imageBlocks.push({ topY: img.topY, markdown: `![${img.id}](${filepath})` });
+					} catch {
+						// Image rendering failed — skip.
+					}
+				}
+			} else if (page.images.length > 0) {
+				for (const img of page.images) {
+					imageBlocks.push({
+						topY: img.topY,
+						markdown: `<!-- image: ${img.id} (page ${img.pageNumber}, ${img.bbox.w}x${img.bbox.h}pt) -->`,
+					});
+				}
+			}
+
+			// Detect column layout.
+			// If the page has vertical segments (tables), suppress column detection
+			// when one detected column is very narrow — that's a table's first column,
+			// not a page layout column.
+			const layout = detectColumns(page.textBoxes);
+			if (layout.columnCount > 1 && page.segments.some(s => Math.abs(s.x1 - s.x2) <= 0.8)) {
+				const pageXMin = Math.min(...page.textBoxes.map(tb => tb.bounds.left));
+				const pageXMax = Math.max(...page.textBoxes.map(tb => tb.bounds.right));
+				const pageWidth = pageXMax - pageXMin;
+				const minColFraction = 0.3;
+				const tooNarrow = layout.columns.some(col => {
+					const colXMin = Math.min(...col.map(tb => tb.bounds.left));
+					const colXMax = Math.max(...col.map(tb => tb.bounds.right));
+					return (colXMax - colXMin) / pageWidth < minColFraction;
+				});
+				if (tooNarrow) {
+					layout.columnCount = 1;
+					layout.columns = [page.textBoxes];
+					layout.boundaries = [];
+				}
+			}
+
+			if (layout.columnCount === 1) {
+				// Single column — process normally.
+				const md = processColumn(page.pageNumber, page.textBoxes, page.segments, imageBlocks);
+				if (md.length > 0) pageMarkdowns.push(md);
+			} else {
+				// Multi-column — process each column independently, then join.
+				const columnMarkdowns: string[] = [];
+				for (const colBoxes of layout.columns) {
+					// Filter segments to those within this column's X range.
+					const colXMin = Math.min(...colBoxes.map(tb => tb.bounds.left));
+					const colXMax = Math.max(...colBoxes.map(tb => tb.bounds.right));
+					const margin = 10;
+					const colSegments = page.segments.filter(seg => {
+						const segXMin = Math.min(seg.x1, seg.x2);
+						const segXMax = Math.max(seg.x1, seg.x2);
+						return segXMax >= colXMin - margin && segXMin <= colXMax + margin;
+					});
+					// Images go with the first column only (no X info to split by).
+					const md = processColumn(
+						page.pageNumber,
+						colBoxes,
+						colSegments,
+						columnMarkdowns.length === 0 ? imageBlocks : [],
+					);
+					if (md.length > 0) columnMarkdowns.push(md);
+				}
+				const joined = columnMarkdowns.join("\n\n");
+				if (joined.length > 0) pageMarkdowns.push(joined);
+			}
+		}
+
+		return { markdown: pageMarkdowns.join("\n\n") };
+	}
+}