@roj-ai/sdk 0.1.16 → 0.1.17

package/dist/plugins/uploads/preprocessors/markitdown-preprocessor.js

@@ -2,11 +2,14 @@
  * Markitdown Preprocessor
  *
  * Converts documents to markdown using Microsoft's markitdown CLI.
- * Supports PDF, DOCX, XLSX, PPTX, HTML, CSV, JSON, XML, EPUB, and more.
+ * Supports DOCX, XLSX, PPTX, HTML, CSV, JSON, XML, EPUB, RTF, ODT.
+ *
+ * PDFs are handled by `PdfPreprocessor` instead — markitdown's PDF backend
+ * (pdfminer.six) is ~20× slower than pdftotext for no real gain on the
+ * mostly-unstructured PDFs we see in practice.
  *
  * Image extraction:
- * - PDF: uses pdfimages (poppler-utils)
- * - DOCX/ODT/EPUB: uses pandoc --extract-media
+ * - DOCX/ODT/EPUB: uses pandoc --extract-media (runs in parallel with markitdown)
  *
  * Extracted images are classified via the image classifier preprocessor.
  * Full content is written to disk; extractedContent contains a structured manifest.
@@ -14,20 +17,31 @@
 import { dirname } from 'node:path';
 import { mapWithConcurrency } from '../../../lib/utils/concurrency.js';
 import { Err, Ok } from '../../../lib/utils/result.js';
-const MAX_IMAGES = 50;
+const MAX_IMAGES = 20;
 const IMAGE_CLASSIFY_CONCURRENCY = 10;
-// markitdown converts a text-only document; even large PDFs finish in seconds.
+/**
+ * Density filter for extracted images. Bytes-per-pixel ratio below this
+ * threshold typically means the image is an alpha mask, overlay layer, or
+ * essentially-empty region — not worth a vision call.
+ *
+ * Empirical reference points:
+ * - Dense photo JPEG: 0.3–1.0 B/px
+ * - Logo / icon PNG: 0.1–0.5 B/px
+ * - Brand PDF layer mask: <0.005 B/px
+ */
+export const MIN_IMAGE_DENSITY_BYTES_PER_PX = 0.05;
+export const MIN_IMAGE_PIXELS = 50 * 50;
+// markitdown converts a text-only document; even large files finish in seconds.
 const MARKITDOWN_TIMEOUT_MS = 60_000;
-// Image extractors (pdfimages, pandoc --extract-media) scale with image count
-// and resolution. Real-world large brand PDFs (40 pages, 5MB images) can take
-// 60–90s. Upload preprocessing is async/background, so allow generous headroom.
+// Image extractors (pandoc --extract-media) scale with image count
+// and resolution. Real-world large docs can take 60–90s. Upload preprocessing
+// is async/background, so allow generous headroom.
 const IMAGE_EXTRACT_TIMEOUT_MS = 5 * 60_000;
 function makeExec(processRunner) {
     return (cmd, args, timeoutMs = MARKITDOWN_TIMEOUT_MS) => processRunner.execFile(cmd, args, { timeout: timeoutMs, maxBuffer: 50 * 1024 * 1024 });
 }
-/** MIME types where markitdown converts to markdown (non-ZIP, non-image) */
+/** MIME types where markitdown converts to markdown (non-ZIP, non-image, non-PDF) */
 const SUPPORTED_MIME_TYPES = [
-    'application/pdf',
     'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
     'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
     'application/vnd.openxmlformats-officedocument.presentationml.presentation',
@@ -52,37 +66,66 @@ const PANDOC_FORMAT_MAP = {
     'application/vnd.oasis.opendocument.text': 'odt',
     'application/epub+zip': 'epub',
 };
-/** MIME types where pdfimages can extract images */
-const PDFIMAGES_MIMES = new Set([
-    'application/pdf',
-]);
 export class MarkitdownPreprocessor {
     name = 'markitdown';
     supportedMimeTypes = SUPPORTED_MIME_TYPES;
     registry;
     logger;
     fs;
+    processRunner;
     exec;
     constructor(config) {
         this.registry = config.registry;
         this.logger = config.logger;
         this.fs = config.fs;
+        this.processRunner = config.process;
         this.exec = makeExec(config.process);
     }
     async process(filePath, mimeType, ctx) {
         const totalStart = Date.now();
-        const derivedPaths = [];
-        const imageEntries = [];
         this.logger.info('Markitdown processing started', { filePath, mimeType });
-        // 1. Convert to markdown via markitdown
         const contentPathResult = ctx.files.realPath('content.md');
         if (!contentPathResult.ok) {
             return Err(new Error('Failed to resolve output path'));
         }
+        await this.fs.mkdir(dirname(contentPathResult.value), { recursive: true });
+        // Race markitdown text conversion and image extraction — they're
+        // independent, so there's no reason to serialize them. For documents
+        // where pandoc extraction isn't applicable, the image task resolves
+        // immediately.
+        const markdownTask = this.runMarkitdown(filePath, mimeType, contentPathResult.value);
+        const imageTask = PANDOC_EXTRACT_MIMES.has(mimeType)
+            ? this.extractImagesWithPandoc(filePath, mimeType, ctx)
+            : Promise.resolve([]);
+        const [markdownResult, images] = await Promise.all([markdownTask, imageTask]);
+        if (!markdownResult.ok)
+            return markdownResult;
+        const markdown = markdownResult.value;
+        const derivedPaths = ['content.md'];
+        const imageEntries = [];
+        for (const img of images) {
+            derivedPaths.push(img.relativePath);
+            imageEntries.push(`- ${img.relativePath} — ${img.description}`);
+        }
+        const manifestLines = ['Extracted files:'];
+        manifestLines.push(`- content.md (markdown, ${markdown.length} chars)`);
+        manifestLines.push(...imageEntries);
+        this.logger.info('Markitdown processing complete', {
+            filePath,
+            mimeType,
+            contentLength: markdown.length,
+            imagesExtracted: imageEntries.length,
+            totalDurationMs: Date.now() - totalStart,
+        });
+        return Ok({
+            extractedContent: manifestLines.join('\n'),
+            derivedPaths,
+        });
+    }
+    async runMarkitdown(filePath, mimeType, outputPath) {
         const markitdownStart = Date.now();
         try {
-            await this.fs.mkdir(dirname(contentPathResult.value), { recursive: true });
-            await this.exec('markitdown', [filePath, '-o', contentPathResult.value]);
+            await this.exec('markitdown', [filePath, '-o', outputPath]);
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);
@@ -92,48 +135,20 @@ export class MarkitdownPreprocessor {
             }
             return Err(new Error(`markitdown failed: ${message}`));
         }
-        const contentResult = await ctx.files.read('content.md');
-        const markdown = contentResult.ok ? contentResult.value : '';
-        derivedPaths.push('content.md');
-        this.logger.info('Markitdown conversion complete', {
-            filePath,
-            mimeType,
-            durationMs: Date.now() - markitdownStart,
-            contentLength: markdown.length,
-        });
-        // 2. Extract images based on file type
-        const imagePhaseStart = Date.now();
-        if (PANDOC_EXTRACT_MIMES.has(mimeType)) {
-            const images = await this.extractImagesWithPandoc(filePath, mimeType, ctx);
-            for (const img of images) {
-                derivedPaths.push(img.relativePath);
-                imageEntries.push(`- ${img.relativePath} — ${img.description}`);
-            }
+        let markdown = '';
+        try {
+            markdown = await this.fs.readFile(outputPath, 'utf-8');
         }
-        else if (PDFIMAGES_MIMES.has(mimeType)) {
-            const images = await this.extractImagesWithPdfimages(filePath, ctx);
-            for (const img of images) {
-                derivedPaths.push(img.relativePath);
-                imageEntries.push(`- ${img.relativePath} — ${img.description}`);
-            }
+        catch {
+            // Output missing — markitdown completed but produced nothing.
         }
-        const imagePhaseDurationMs = Date.now() - imagePhaseStart;
-        // 3. Build manifest
-        const manifestLines = ['Extracted files:'];
-        manifestLines.push(`- content.md (markdown, ${markdown.length} chars)`);
-        manifestLines.push(...imageEntries);
-        this.logger.info('Markitdown processing complete', {
+        this.logger.info('Markitdown conversion complete', {
             filePath,
             mimeType,
+            durationMs: Date.now() - markitdownStart,
             contentLength: markdown.length,
-            imagesExtracted: imageEntries.length,
-            imagePhaseDurationMs,
-            totalDurationMs: Date.now() - totalStart,
-        });
-        return Ok({
-            extractedContent: manifestLines.join('\n'),
-            derivedPaths,
         });
+        return Ok(markdown);
     }
     async extractImagesWithPandoc(filePath, mimeType, ctx) {
         const mediaStore = ctx.files.scoped('media');
@@ -164,7 +179,7 @@ export class MarkitdownPreprocessor {
             });
         }
         const classifyStart = Date.now();
-        const images = await classifyExtractedImages(mediaStore, 'media', ctx, this.registry, this.logger);
+        const images = await classifyExtractedImages(mediaStore, 'media', ctx, this.registry, this.logger, this.fs, this.processRunner);
         this.logger.info('Image classification complete', {
             source: 'pandoc',
             count: images.length,
@@ -173,46 +188,16 @@ export class MarkitdownPreprocessor {
         });
         return images;
     }
-    async extractImagesWithPdfimages(filePath, ctx) {
-        const imageStore = ctx.files.scoped('images');
-        const imagesDirResult = imageStore.realPath('');
-        if (!imagesDirResult.ok)
-            return [];
-        const pdfimagesStart = Date.now();
-        let extractSucceeded = true;
-        try {
-            await this.fs.mkdir(imagesDirResult.value, { recursive: true });
-            await this.exec('pdfimages', ['-png', filePath, `${imagesDirResult.value}/img`], IMAGE_EXTRACT_TIMEOUT_MS);
-        }
-        catch (error) {
-            extractSucceeded = false;
-            this.logger.warn('pdfimages failed (will classify any partial output)', {
-                filePath,
-                durationMs: Date.now() - pdfimagesStart,
-                error: error instanceof Error ? error.message : String(error),
-            });
-        }
-        if (extractSucceeded) {
-            this.logger.info('pdfimages complete', {
-                filePath,
-                durationMs: Date.now() - pdfimagesStart,
-            });
-        }
-        const classifyStart = Date.now();
-        const images = await classifyExtractedImages(imageStore, 'images', ctx, this.registry, this.logger);
-        this.logger.info('Image classification complete', {
-            source: 'pdfimages',
-            count: images.length,
-            partial: !extractSucceeded,
-            durationMs: Date.now() - classifyStart,
-        });
-        return images;
-    }
 }
 // ============================================================================
 // Shared image helpers
 // ============================================================================
-const IMAGE_EXT_RE = /\.(png|jpe?g|gif|webp|tiff?|bmp|svg)$/i;
+/**
+ * Recognized by Anthropic vision API. Other pdfimages outputs (pbm, ppm,
+ * jb2e, jp2) are ignored — they'd require local conversion before being
+ * useful for classification.
+ */
+export const IMAGE_EXT_RE = /\.(png|jpe?g|gif|webp|tiff?|bmp|svg)$/i;
 const IMAGE_MIME_MAP = {
     png: 'image/png',
     jpg: 'image/jpeg',
@@ -228,15 +213,85 @@ export function guessImageMime(filename) {
     const ext = filename.split('.').pop()?.toLowerCase();
     return IMAGE_MIME_MAP[ext ?? ''] ?? 'image/png';
 }
-export async function classifyExtractedImages(imageStore, relativePrefix, ctx, registry, logger) {
+/**
+ * Reject images that are unlikely to carry useful visual information.
+ *
+ * `bytesPerPixel` filters out alpha masks, sparse overlays, and essentially-
+ * empty pages — brand PDFs typically emit a real photo (~1 B/px) plus a
+ * matching transparency/overlay layer at the same dimensions but a fraction
+ * of a percent of the size (<0.005 B/px).
+ *
+ * The minimum pixel count protects against tiny icons whose density alone
+ * doesn't disqualify them.
+ */
+export function shouldClassifyImage(meta) {
+    const pixels = meta.width * meta.height;
+    if (pixels < MIN_IMAGE_PIXELS)
+        return false;
+    const density = meta.sizeBytes / pixels;
+    return density >= MIN_IMAGE_DENSITY_BYTES_PER_PX;
+}
+/**
+ * Read image dimensions via vipsheader. Returns null when the tool isn't
+ * available or output is unparseable — caller should treat that as
+ * "include without filtering".
+ */
+export async function getImageDimensions(filePath, processRunner) {
+    try {
+        const { stdout } = await processRunner.execFile('vipsheader', ['-f', 'width', '-f', 'height', filePath], { timeout: 10_000 });
+        const lines = stdout.trim().split('\n');
+        if (lines.length < 2)
+            return null;
+        const width = parseInt(lines[0], 10);
+        const height = parseInt(lines[1], 10);
+        if (!Number.isFinite(width) || !Number.isFinite(height))
+            return null;
+        return { width, height };
+    }
+    catch {
+        return null;
+    }
+}
+export async function classifyExtractedImages(imageStore, relativePrefix, ctx, registry, logger, fs, processRunner) {
     const listResult = await imageStore.list('', { maxDepth: 3 });
     if (!listResult.ok)
         return [];
-    const imageFiles = listResult.value
-        .filter(e => e.type === 'file' && IMAGE_EXT_RE.test(e.name))
-        .sort((a, b) => a.name.localeCompare(b.name))
+    const candidates = listResult.value.filter(e => e.type === 'file' && IMAGE_EXT_RE.test(e.name));
+    // Stat + density filter, then keep the top MAX_IMAGES by file size.
+    const inspected = await mapWithConcurrency(candidates, 8, async (entry) => {
+        const pathResult = imageStore.realPath(entry.name);
+        if (!pathResult.ok)
+            return null;
+        let sizeBytes = 0;
+        try {
+            sizeBytes = (await fs.stat(pathResult.value)).size;
+        }
+        catch {
+            return null;
+        }
+        const dims = await getImageDimensions(pathResult.value, processRunner);
+        if (!dims) {
+            // Unknown dims — include but warn; better to classify than silently drop.
+            return { name: entry.name, sizeBytes, width: 0, height: 0, kept: true };
+        }
+        const kept = shouldClassifyImage({ width: dims.width, height: dims.height, sizeBytes });
+        return { name: entry.name, sizeBytes, width: dims.width, height: dims.height, kept };
+    });
+    const filtered = inspected
+        .filter((r) => r !== null && r.kept)
+        .sort((a, b) => b.sizeBytes - a.sizeBytes)
         .slice(0, MAX_IMAGES);
-    const settled = await mapWithConcurrency(imageFiles, IMAGE_CLASSIFY_CONCURRENCY, async (imgFile) => {
+    const droppedCount = inspected.filter(r => r !== null && !r.kept).length;
+    if (droppedCount > 0 || inspected.length > MAX_IMAGES) {
+        logger.info('Image filter applied', {
+            source: relativePrefix,
+            candidates: candidates.length,
+            passedDensityFilter: candidates.length - droppedCount,
+            selected: filtered.length,
+            droppedByDensity: droppedCount,
+        });
+    }
+    const settled = await mapWithConcurrency(filtered, IMAGE_CLASSIFY_CONCURRENCY, async (imgFile) => {
         const imgPathResult = imageStore.realPath(imgFile.name);
         if (!imgPathResult.ok)
             return null;

package/dist/plugins/uploads/preprocessors/markitdown-preprocessor.js.map

@@ -1 +1 @@
-{"version":3,"file":"markitdown-preprocessor.js","sourceRoot":"","sources":["../../../../src/plugins/uploads/preprocessors/markitdown-preprocessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAE/D,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,uBAAuB,CAAA;AAO/C,MAAM,UAAU,GAAG,EAAE,CAAA;AACrB,MAAM,0BAA0B,GAAG,EAAE,CAAA;AAErC,+EAA+E;AAC/E,MAAM,qBAAqB,GAAG,MAAM,CAAA;AACpC,8EAA8E;AAC9E,8EAA8E;AAC9E,gFAAgF;AAChF,MAAM,wBAAwB,GAAG,CAAC,GAAG,MAAM,CAAA;AAE3C,SAAS,QAAQ,CAAC,aAA4B;IAC7C,OAAO,CAAC,GAAW,EAAE,IAAc,EAAE,YAAoB,qBAAqB,EAAE,EAAE,CACjF,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAA;AACxF,CAAC;AAED,4EAA4E;AAC5E,MAAM,oBAAoB,GAAG;IAC5B,iBAAiB;IACjB,yEAAyE;IACzE,mEAAmE;IACnE,2EAA2E;IAC3E,yCAAyC;IACzC,iBAAiB;IACjB,sBAAsB;IACtB,WAAW;IACX,uBAAuB;IACvB,UAAU;IACV,kBAAkB;IAClB,iBAAiB;IACjB,UAAU;CACV,CAAA;AAED,yDAAyD;AACzD,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACpC,yEAAyE;IACzE,yCAAyC;IACzC,sBAAsB;CACtB,CAAC,CAAA;AAEF,MAAM,iBAAiB,GAA2B;IACjD,yEAAyE,EAAE,MAAM;IACjF,yCAAyC,EAAE,KAAK;IAChD,sBAAsB,EAAE,MAAM;CAC9B,CAAA;AAED,oDAAoD;AACpD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC/B,iBAAiB;CACjB,CAAC,CAAA;AASF,MAAM,OAAO,sBAAsB;IACzB,IAAI,GAAG,YAAY,CAAA;IACnB,kBAAkB,GAAG,oBAAoB,CAAA;IAEjC,QAAQ,CAAsB;IAC9B,MAAM,CAAQ;IACd,EAAE,CAAY;IACd,IAAI,CAAkG;IAEvH,YAAY,MAAoC;QAC/C,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAA;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;QAC3B,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAA;QACnB,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,OAAO,CACZ,QAAgB,EAChB,QAAgB,EAChB,GAAwB;QAExB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAC7B,MAAM,YAAY,GAAa,EAAE,CAAA;QACjC,MAAM,YAAY,GAAa,EAAE,CAAA;QAEjC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;QAEzE,wCAAwC;QACxC,MAAM,iBAAiB,GAAG,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;QAC1D,IAAI,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC;YAC3B,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC,CAAA;QACvD,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAClC,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YAC1E,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAA;QACzE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,IAAI,CAAC,MAAM,CAAC,KAAK,CAChB,uBAAuB,EACvB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,EAC1C,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,EAAE,CAChE,CAAA;YACD,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAChC,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC,CAAA;YAC3F,CAAC;YACD,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,sBAAsB,OAAO,EAAE,CAAC,CAAC,CAAA;QACvD,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;QACxD,MAAM,QAAQ,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAA;QAE5D,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;QAE/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;YAClD,QAAQ;YACR,QAAQ;YACR,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe;YACxC,aAAa,EAAE,QAAQ,CAAC,MAAM;SAC9B,CAAC,CAAA;QAEF,uCAAuC;QACvC,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAClC,IAAI,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAA;YAC1E,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAC1B,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;gBACnC,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,YAAY,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YAChE,CAAC;QACF,CAAC;aAAM,IAAI,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;YACnE,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAC1B,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;gBACnC,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,YAAY,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YAChE,CAAC;QACF,CAAC;QACD,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAA;QAEzD,oBAAoB;QACpB,MAAM,aAAa,GAAa,CAAC,kBAAkB,CAAC,CAAA;QACpD,aAAa,CAAC,IAAI,CAAC,2BAA2B,QAAQ,CAAC,MAAM,SAAS,CAAC,CAAA;QACvE,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAA;QAEnC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;YAClD,QAAQ;YACR,QAAQ;YACR,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,eAAe,EAAE,YAAY,CAAC,MAAM;YACpC,oBAAoB;YACpB,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;SACxC,CAAC,CAAA;QAEF,OAAO,EAAE,CAAC;YACT,gBAAgB,EAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1C,YAAY;SACZ,CAAC,CAAA;IACH,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACpC,QAAgB,EAChB,QAAgB,EAChB,GAAwB;QAExB,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;QAC9C,IAAI,CAAC,cAAc,CAAC,EAAE;YAAE,OAAO,EAAE,CAAA;QAEjC,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QAC1C,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,CAAA;QAEtB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAC9B,IAAI,gBAAgB,GAAG,IAAI,CAAA;QAC3B,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,IAAI,CACd,QAAQ,EACR,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,mBAAmB,cAAc,CAAC,KAAK,EAAE,CAAC,EACnG,wBAAwB,CACxB,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,gBAAgB,GAAG,KAAK,CAAA;YACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kEAAkE,EAAE;gBACpF,QAAQ;gBACR,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW;gBACpC,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC7D,CAAC,CAAA;QACH,CAAC;QACD,IAAI,gBAAgB,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE;gBACnD,QAAQ;gBACR,MAAM;gBACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW;aACpC,CAAC,CAAA;QACH,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAChC,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAClG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;YACjD,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,OAAO,EAAE,CAAC,gBAAgB;YAC1B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa;SACtC,CAAC,CAAA;QACF,OAAO,MAAM,CAAA;IACd,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACvC,QAAgB,EAChB,GAAwB;QAExB,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAC7C,MAAM,eAAe,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;QAC/C,IAAI,CAAC,eAAe,CAAC,EAAE;YAAE,OAAO,EAAE,CAAA;QAElC,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACjC,IAAI,gBAAgB,GAAG,IAAI,CAAA;QAC3B,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YAC/D,MAAM,IAAI,CAAC,IAAI,CACd,WAAW,EACX,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC,KAAK,MAAM,CAAC,EAClD,wBAAwB,CACxB,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,gBAAgB,GAAG,KAAK,CAAA;YACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qDAAqD,EAAE;gBACvE,QAAQ;gBACR,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;gBACvC,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC7D,CAAC,CAAA;QACH,CAAC;QACD,IAAI,gBAAgB,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE;gBACtC,QAAQ;gBACR,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;aACvC,CAAC,CAAA;QACH,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAChC,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QACnG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;YACjD,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,OAAO,EAAE,CAAC,gBAAgB;YAC1B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa;SACtC,CAAC,CAAA;QACF,OAAO,MAAM,CAAA;IACd,CAAC;CACD;AAED,+EAA+E;AAC/E,uBAAuB;AACvB,+EAA+E;AAE/E,MAAM,YAAY,GAAG,wCAAwC,CAAA;AAE7D,MAAM,cAAc,GAA2B;IAC9C,GAAG,EAAE,WAAW;IAChB,GAAG,EAAE,YAAY;IACjB,IAAI,EAAE,YAAY;IAClB,GAAG,EAAE,WAAW;IAChB,IAAI,EAAE,YAAY;IAClB,GAAG,EAAE,eAAe;IACpB,GAAG,EAAE,YAAY;IACjB,IAAI,EAAE,YAAY;IAClB,GAAG,EAAE,WAAW;CAChB,CAAA;AAED,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC9C,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAA;IACpD,OAAO,cAAc,CAAC,GAAG,IAAI,EAAE,CAAC,IAAI,WAAW,CAAA;AAChD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC5C,UAAqB,EACrB,cAAsB,EACtB,GAAwB,EACxB,QAA8B,EAC9B,MAAc;IAEd,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAA;IAC7D,IAAI,CAAC,UAAU,CAAC,EAAE;QAAE,OAAO,EAAE,CAAA;IAE7B,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK;SACjC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAC3D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAC5C,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAA;IAEtB,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,UAAU,EAAE,0BAA0B,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAClG,MAAM,aAAa,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QACvD,IAAI,CAAC,aAAa,CAAC,EAAE;YAAE,OAAO,IAAI,CAAA;QAElC,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,WAAW,GAAG,OAAO,CAAA;QAEzB,MAAM,UAAU,GAAG,QAAQ,CAAC,cAAc,CAAC,OAAO,CAAC,CAAA;QACnD,IAAI,UAAU,EAAE,CAAC;YAChB,MAAM,cAAc,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE;gBAC7E,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,cAAc,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC;aACjE,CAAC,CAAA;YACF,IAAI,cAAc,CAAC,EAAE,IAAI,cAAc,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;gBAChE,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,gBAAgB,CAAA;YACpD,CAAC;QACF,CAAC;QAED,OAAO,EAAE,YAAY,EAAE,GAAG,cAAc,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,CAAA;IAC1E,CAAC,CAAC,CAAA;IAEF,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAsD,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;AAC7F,CAAC"}
+{"version":3,"file":"markitdown-preprocessor.js","sourceRoot":"","sources":["../../../../src/plugins/uploads/preprocessors/markitdown-preprocessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAE/D,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,uBAAuB,CAAA;AAO/C,MAAM,UAAU,GAAG,EAAE,CAAA;AACrB,MAAM,0BAA0B,GAAG,EAAE,CAAA;AAErC;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,IAAI,CAAA;AAClD,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,GAAG,EAAE,CAAA;AAEvC,gFAAgF;AAChF,MAAM,qBAAqB,GAAG,MAAM,CAAA;AACpC,mEAAmE;AACnE,8EAA8E;AAC9E,mDAAmD;AACnD,MAAM,wBAAwB,GAAG,CAAC,GAAG,MAAM,CAAA;AAE3C,SAAS,QAAQ,CAAC,aAA4B;IAC7C,OAAO,CAAC,GAAW,EAAE,IAAc,EAAE,YAAoB,qBAAqB,EAAE,EAAE,CACjF,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAA;AACxF,CAAC;AAED,qFAAqF;AACrF,MAAM,oBAAoB,GAAG;IAC5B,yEAAyE;IACzE,mEAAmE;IACnE,2EAA2E;IAC3E,yCAAyC;IACzC,iBAAiB;IACjB,sBAAsB;IACtB,WAAW;IACX,uBAAuB;IACvB,UAAU;IACV,kBAAkB;IAClB,iBAAiB;IACjB,UAAU;CACV,CAAA;AAED,yDAAyD;AACzD,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACpC,yEAAyE;IACzE,yCAAyC;IACzC,sBAAsB;CACtB,CAAC,CAAA;AAEF,MAAM,iBAAiB,GAA2B;IACjD,yEAAyE,EAAE,MAAM;IACjF,yCAAyC,EAAE,KAAK;IAChD,sBAAsB,EAAE,MAAM;CAC9B,CAAA;AASD,MAAM,OAAO,sBAAsB;IACzB,IAAI,GAAG,YAAY,CAAA;IACnB,kBAAkB,GAAG,oBAAoB,CAAA;IAEjC,QAAQ,CAAsB;IAC9B,MAAM,CAAQ;IACd,EAAE,CAAY;IACd,aAAa,CAAe;IAC5B,IAAI,CAAkG;IAEvH,YAAY,MAAoC;QAC/C,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAA;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;QAC3B,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAA;QACnB,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,OAAO,CAAA;QACnC,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,OAAO,CACZ,QAAgB,EAChB,QAAgB,EAChB,GAAwB;QAExB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;QAEzE,MAAM,iBAAiB,GAAG,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;QAC1D,IAAI,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC;YAC3B,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC,CAAA;QACvD,CAAC;QACD,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAE1E,iEAAiE;QACjE,qEAAqE;QACrE,oEAAoE;QACpE,eAAe;QACf,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,CAAC,KAAK,CAAC,CAAA;QACpF,MAAM,SAAS,GAAG,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC;YACnD,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC;YACvD,CAAC,CAAC,OAAO,CAAC,OAAO,CAAuD,EAAE,CAAC,CAAA;QAE5E,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC,CAAA;QAE7E,IAAI,CAAC,cAAc,CAAC,EAAE;YAAE,OAAO,cAAc,CAAA;QAE7C,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAA;QAErC,MAAM,YAAY,GAAa,CAAC,YAAY,CAAC,CAAA;QAC7C,MAAM,YAAY,GAAa,EAAE,CAAA;QACjC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YAC1B,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;YACnC,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,YAAY,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;QAChE,CAAC;QAED,MAAM,aAAa,GAAa,CAAC,kBAAkB,CAAC,CAAA;QACpD,aAAa,CAAC,IAAI,CAAC,2BAA2B,QAAQ,CAAC,MAAM,SAAS,CAAC,CAAA;QACvE,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAA;QAEnC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;YAClD,QAAQ;YACR,QAAQ;YACR,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,eAAe,EAAE,YAAY,CAAC,MAAM;YACpC,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;SACxC,CAAC,CAAA;QAEF,OAAO,EAAE,CAAC;YACT,gBAAgB,EAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1C,YAAY;SACZ,CAAC,CAAA;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAC1B,QAAgB,EAChB,QAAgB,EAChB,UAAkB;QAElB,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAClC,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAA;QAC5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,IAAI,CAAC,MAAM,CAAC,KAAK,CAChB,uBAAuB,EACvB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,EAC1C,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,EAAE,CAChE,CAAA;YACD,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAChC,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC,CAAA;YAC3F,CAAC;YACD,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,sBAAsB,OAAO,EAAE,CAAC,CAAC,CAAA;QACvD,CAAC;QAED,IAAI,QAAQ,GAAG,EAAE,CAAA;QACjB,IAAI,CAAC;YACJ,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;QACvD,CAAC;QAAC,MAAM,CAAC;YACR,8DAA8D;QAC/D,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;YAClD,QAAQ;YACR,QAAQ;YACR,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe;YACxC,aAAa,EAAE,QAAQ,CAAC,MAAM;SAC9B,CAAC,CAAA;QAEF,OAAO,EAAE,CAAC,QAAQ,CAAC,CAAA;IACpB,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACpC,QAAgB,EAChB,QAAgB,EAChB,GAAwB;QAExB,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;QAC9C,IAAI,CAAC,cAAc,CAAC,EAAE;YAAE,OAAO,EAAE,CAAA;QAEjC,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QAC1C,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,CAAA;QAEtB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAC9B,IAAI,gBAAgB,GAAG,IAAI,CAAA;QAC3B,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,IAAI,CACd,QAAQ,EACR,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,mBAAmB,cAAc,CAAC,KAAK,EAAE,CAAC,EACnG,wBAAwB,CACxB,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,gBAAgB,GAAG,KAAK,CAAA;YACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kEAAkE,EAAE;gBACpF,QAAQ;gBACR,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW;gBACpC,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC7D,CAAC,CAAA;QACH,CAAC;QACD,IAAI,gBAAgB,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE;gBACnD,QAAQ;gBACR,MAAM;gBACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW;aACpC,CAAC,CAAA;QACH,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAChC,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAC/H,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;YACjD,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,OAAO,EAAE,CAAC,gBAAgB;YAC1B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa;SACtC,CAAC,CAAA;QACF,OAAO,MAAM,CAAA;IACd,CAAC;CACD;AAED,+EAA+E;AAC/E,uBAAuB;AACvB,+EAA+E;AAE/E;;;;GAIG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,wCAAwC,CAAA;AAEpE,MAAM,cAAc,GAA2B;IAC9C,GAAG,EAAE,WAAW;IAChB,GAAG,EAAE,YAAY;IACjB,IAAI,EAAE,YAAY;IAClB,GAAG,EAAE,WAAW;IAChB,IAAI,EAAE,YAAY;IAClB,GAAG,EAAE,eAAe;IACpB,GAAG,EAAE,YAAY;IACjB,IAAI,EAAE,YAAY;IAClB,GAAG,EAAE,WAAW;CAChB,CAAA;AAED,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC9C,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAA;IACpD,OAAO,cAAc,CAAC,GAAG,IAAI,EAAE,CAAC,IAAI,WAAW,CAAA;AAChD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAA0D;IAC7F,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAA;IACvC,IAAI,MAAM,GAAG,gBAAgB;QAAE,OAAO,KAAK,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,GAAG,MAAM,CAAA;IACvC,OAAO,OAAO,IAAI,8BAA8B,CAAA;AACjD,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,QAAgB,EAChB,aAA4B;IAE5B,IAAI,CAAC;QACJ,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,QAAQ,CAC9C,YAAY,EACZ,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,EACzC,EAAE,OAAO,EAAE,MAAM,EAAE,CACnB,CAAA;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACvC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,IAAI,CAAA;QACjC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACrC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAA;QACpE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;IACzB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC5C,UAAqB,EACrB,cAAsB,EACtB,GAAwB,EACxB,QAA8B,EAC9B,MAAc,EACd,EAAc,EACd,aAA4B;IAE5B,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAA;IAC7D,IAAI,CAAC,UAAU,CAAC,EAAE;QAAE,OAAO,EAAE,CAAA;IAE7B,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAE/F,oEAAoE;IACpE,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QACzE,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAClD,IAAI,CAAC,UAAU,CAAC,EAAE;YAAE,OAAO,IAAI,CAAA;QAE/B,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,CAAC;YACJ,SAAS,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;QACnD,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,IAAI,CAAA;QACZ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,UAAU,CAAC,KAAK,EAAE,aAAa,CAAC,CAAA;QACtE,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,0EAA0E;YAC1E,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,mBAAmB,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;QACvF,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAA;IACrF,CAAC,CAAC,CAAA;IAEF,MAAM,QAAQ,GAAG,SAAS;SACxB,MAAM,CAAC,CAAC,CAAC,EAA8B,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC;SAC/D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;SACzC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAA;IAEtB,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAA;IACxE,IAAI,YAAY,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;QACvD,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;YACnC,MAAM,EAAE,cAAc;YACtB,UAAU,EAAE,UAAU,CAAC,MAAM;YAC7B,mBAAmB,EAAE,UAAU,CAAC,MAAM,GAAG,YAAY;YACrD,QAAQ,EAAE,QAAQ,CAAC,MAAM;YACzB,gBAAgB,EAAE,YAAY;SAC9B,CAAC,CAAA;IACH,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,0BAA0B,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAChG,MAAM,aAAa,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QACvD,IAAI,CAAC,aAAa,CAAC,EAAE;YAAE,OAAO,IAAI,CAAA;QAElC,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,WAAW,GAAG,OAAO,CAAA;QAEzB,MAAM,UAAU,GAAG,QAAQ,CAAC,cAAc,CAAC,OAAO,CAAC,CAAA;QACnD,IAAI,UAAU,EAAE,CAAC;YAChB,MAAM,cAAc,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE;gBAC7E,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,cAAc,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC;aACjE,CAAC,CAAA;YACF,IAAI,cAAc,CAAC,EAAE,IAAI,cAAc,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;gBAChE,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,gBAAgB,CAAA;YACpD,CAAC;QACF,CAAC;QAED,OAAO,EAAE,YAAY,EAAE,GAAG,cAAc,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,CAAA;IAC1E,CAAC,CAAC,CAAA;IAEF,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAsD,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;AAC7F,CAAC"}

package/dist/plugins/uploads/preprocessors/pdf-preprocessor.d.ts

@@ -0,0 +1,71 @@
+/**
+ * PDF Preprocessor
+ *
+ * Dedicated PDF pipeline:
+ *
+ * 1. Text extraction via `pdftotext` (poppler-utils, C++) — ~1 s for a
+ *    3 MB PDF. Replaces markitdown/pdfminer.six (~22 s for the same file)
+ *    because PDFs in practice don't carry the rich markdown structure
+ *    that justifies the slower backend.
+ *
+ * 2. Image extraction via `pdfimages -all` — keeps the original embedded
+ *    format (JPEG stays JPEG) instead of re-encoding everything to PNG
+ *    (~10× faster, much smaller files).
+ *
+ * 3. Text and image extraction run in parallel.
+ *
+ * 4. Images stream into the classifier as soon as `pdfimages` writes them
+ *    to disk — the classifier doesn't wait for the whole extraction to
+ *    finish. A density filter (bytes/pixel) drops alpha masks and overlay
+ *    layers before the vision call.
+ */
+import type { Result } from '../../../lib/utils/result.js';
+import type { FileSystem } from '../../../platform/fs.js';
+import type { ProcessRunner } from '../../../platform/process.js';
+import type { Logger } from '../../../lib/logger/logger.js';
+import type { Preprocessor, PreprocessorContext, PreprocessorRegistry, PreprocessorResult } from '../preprocessor.js';
+export interface PdfPreprocessorConfig {
+    registry: PreprocessorRegistry;
+    logger: Logger;
+    fs: FileSystem;
+    process: ProcessRunner;
+}
+export declare class PdfPreprocessor implements Preprocessor {
+    readonly name = "pdf";
+    readonly supportedMimeTypes: string[];
+    private readonly registry;
+    private readonly logger;
+    private readonly fs;
+    private readonly processRunner;
+    constructor(config: PdfPreprocessorConfig);
+    process(filePath: string, mimeType: string, ctx: PreprocessorContext): Promise<Result<PreprocessorResult, Error>>;
+    /**
+     * Extract plain text via pdftotext. Writes to content.md verbatim — no
+     * markdown structure to preserve, but the file extension stays .md for
+     * consistency with the markitdown pipeline (downstream consumers expect
+     * "content.md" in the upload directory).
+     *
+     * `-layout` preserves the original visual layout (columns, tables),
+     * which is what users typically expect when looking at PDFs.
+     */
+    private extractText;
+    /**
+     * Extract images via pdfimages and classify them as they appear on disk.
+     *
+     * pdfimages writes files atomically per image (open temp, write, rename
+     * to final name), so polling `readdir` is safe — we either see a name or
+     * we don't, never a half-written file.
+     *
+     * Streaming overlaps the extraction tail with the first classification
+     * batches. Hard cap of MAX_IMAGES applies across the *filtered* set: as
+     * soon as MAX_IMAGES images have passed the density filter, further
+     * candidates are stat-checked but not classified.
+     *
+     * `-all` keeps the embedded format (JPEG, JBIG2, JP2). We only classify
+     * those Anthropic vision accepts (PNG/JPEG/GIF/WebP); other formats are
+     * extracted to disk for reference but skipped at the classification step.
+     */
+    private extractAndClassifyImages;
+    private scanAndDispatch;
+}
+//# sourceMappingURL=pdf-preprocessor.d.ts.map

package/dist/plugins/uploads/preprocessors/pdf-preprocessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-preprocessor.d.ts","sourceRoot":"","sources":["../../../../src/plugins/uploads/preprocessors/pdf-preprocessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAEnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAClD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAC1D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,+BAA+B,CAAA;AAC3D,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AAmBrH,MAAM,WAAW,qBAAqB;IACrC,QAAQ,EAAE,oBAAoB,CAAA;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,EAAE,EAAE,UAAU,CAAA;IACd,OAAO,EAAE,aAAa,CAAA;CACtB;AAED,qBAAa,eAAgB,YAAW,YAAY;IACnD,QAAQ,CAAC,IAAI,SAAQ;IACrB,QAAQ,CAAC,kBAAkB,WAAuB;IAElD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAY;IAC/B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAe;gBAEjC,MAAM,EAAE,qBAAqB;IAOnC,OAAO,CACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,mBAAmB,GACtB,OAAO,CAAC,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC,CAAC;IA8C7C;;;;;;;;OAQG;YACW,WAAW;IA8BzB;;;;;;;;;;;;;;;OAeG;YACW,wBAAwB;YAoJxB,eAAe;CAc7B"}