kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/mcp/tools/mime.rs

@@ -0,0 +1,150 @@
+//! MIME type detection MCP tool.
+
+use crate::{detect_mime_type, mcp::errors::map_kreuzberg_error_to_mcp, mcp::params::DetectMimeTypeParams};
+use rmcp::{
+    ErrorData as McpError,
+    handler::server::wrapper::Parameters,
+    model::{CallToolResult, Content, RawContent},
+    tool,
+};
+
+/// MCP tool methods for MIME type detection.
+pub(in crate::mcp) trait MimeTypeTool {
+    /// Detect the MIME type of a file.
+    ///
+    /// This tool identifies the file format, useful for determining which extractor to use.
+    #[tool(
+        description = "Detect the MIME type of a file. Returns the detected MIME type string.",
+        annotations(title = "Detect MIME Type", read_only_hint = true, idempotent_hint = true)
+    )]
+    fn detect_mime_type(
+        &self,
+        Parameters(params): Parameters<DetectMimeTypeParams>,
+    ) -> Result<CallToolResult, McpError> {
+        let mime_type = detect_mime_type(&params.path, params.use_content).map_err(map_kreuzberg_error_to_mcp)?;
+
+        Ok(CallToolResult::success(vec![Content::text(mime_type)]))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::ExtractionConfig;
+    use std::path::PathBuf;
+
+    /// Get the path to a test document relative to workspace root.
+    fn get_test_path(relative_path: &str) -> String {
+        let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .parent()
+            .unwrap()
+            .parent()
+            .unwrap()
+            .to_path_buf();
+
+        workspace_root
+            .join("test_documents")
+            .join(relative_path)
+            .to_string_lossy()
+            .to_string()
+    }
+
+    // Simple test struct for trait implementation
+    struct TestMcpServer;
+
+    impl MimeTypeTool for TestMcpServer {}
+
+    #[tokio::test]
+    async fn test_detect_mime_type_with_valid_file() {
+        let server = TestMcpServer;
+        let params = DetectMimeTypeParams {
+            path: get_test_path("pdfs_with_tables/tiny.pdf").to_string(),
+            use_content: true,
+        };
+
+        let result = server.detect_mime_type(Parameters(params));
+
+        assert!(result.is_ok());
+        let call_result = result.unwrap();
+        if let Some(content) = call_result.content.first() {
+            match &content.raw {
+                RawContent::Text(text) => {
+                    assert!(text.text.contains("application/pdf") || text.text.contains("pdf"));
+                }
+                _ => panic!("Expected text content"),
+            }
+        } else {
+            panic!("Expected content in result");
+        }
+    }
+
+    #[tokio::test]
+    async fn test_detect_mime_type_without_content_detection() {
+        let server = TestMcpServer;
+        let params = DetectMimeTypeParams {
+            path: get_test_path("pdfs_with_tables/tiny.pdf").to_string(),
+            use_content: false,
+        };
+
+        let result = server.detect_mime_type(Parameters(params));
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_detect_mime_type_with_invalid_file() {
+        let server = TestMcpServer;
+        let params = DetectMimeTypeParams {
+            path: "/nonexistent/file.pdf".to_string(),
+            use_content: true,
+        };
+
+        let result = server.detect_mime_type(Parameters(params));
+
+        assert!(result.is_err());
+        let error = result.unwrap_err();
+        assert!(error.code.0 == -32602 || error.code.0 == -32603);
+    }
+
+    #[tokio::test]
+    async fn test_detect_mime_type_with_extension_only() {
+        let server = TestMcpServer;
+
+        let test_file = get_test_path("pdfs_with_tables/tiny.pdf");
+
+        if std::path::Path::new(&test_file).exists() {
+            let params = DetectMimeTypeParams {
+                path: test_file.to_string(),
+                use_content: false,
+            };
+
+            let result = server.detect_mime_type(Parameters(params));
+
+            assert!(result.is_ok());
+            let call_result = result.unwrap();
+            if let Some(content) = call_result.content.first()
+                && let RawContent::Text(text) = &content.raw
+            {
+                assert!(text.text.contains("pdf") || text.text.contains("PDF"));
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_detect_mime_type_with_content_analysis() {
+        let server = TestMcpServer;
+
+        let test_file = get_test_path("pdfs_with_tables/tiny.pdf");
+
+        if std::path::Path::new(&test_file).exists() {
+            let params = DetectMimeTypeParams {
+                path: test_file.to_string(),
+                use_content: true,
+            };
+
+            let result = server.detect_mime_type(Parameters(params));
+
+            assert!(result.is_ok());
+        }
+    }
+}

data/vendor/kreuzberg/src/mcp/tools/mod.rs

@@ -0,0 +1,11 @@
+//! MCP tool implementations.
+//!
+//! This module organizes MCP tools by category: extraction, cache management, and MIME detection.
+
+mod cache;
+mod extraction;
+mod mime;
+
+pub(in crate::mcp) use cache::CacheTool;
+pub(in crate::mcp) use extraction::ExtractionTool;
+pub(in crate::mcp) use mime::MimeTypeTool;

data/vendor/kreuzberg/src/ocr/backends/easyocr.rs

@@ -0,0 +1,96 @@
+//! EasyOCR language support.
+//!
+//! EasyOCR supports 83 languages with broad multilingual coverage.
+
+/// Get list of languages supported by EasyOCR.
+///
+/// # Returns
+///
+/// A vector of 83 language codes supported by EasyOCR.
+pub(in crate::ocr) fn languages() -> Vec<String> {
+    vec![
+        "abq".to_string(),
+        "ady".to_string(),
+        "af".to_string(),
+        "ang".to_string(),
+        "ar".to_string(),
+        "as".to_string(),
+        "ava".to_string(),
+        "az".to_string(),
+        "be".to_string(),
+        "bg".to_string(),
+        "bh".to_string(),
+        "bho".to_string(),
+        "bn".to_string(),
+        "bs".to_string(),
+        "ch_sim".to_string(),
+        "ch_tra".to_string(),
+        "che".to_string(),
+        "cs".to_string(),
+        "cy".to_string(),
+        "da".to_string(),
+        "dar".to_string(),
+        "de".to_string(),
+        "en".to_string(),
+        "es".to_string(),
+        "et".to_string(),
+        "fa".to_string(),
+        "fr".to_string(),
+        "ga".to_string(),
+        "gom".to_string(),
+        "hi".to_string(),
+        "hr".to_string(),
+        "hu".to_string(),
+        "id".to_string(),
+        "inh".to_string(),
+        "is".to_string(),
+        "it".to_string(),
+        "ja".to_string(),
+        "kbd".to_string(),
+        "kn".to_string(),
+        "ko".to_string(),
+        "ku".to_string(),
+        "la".to_string(),
+        "lbe".to_string(),
+        "lez".to_string(),
+        "lt".to_string(),
+        "lv".to_string(),
+        "mah".to_string(),
+        "mai".to_string(),
+        "mi".to_string(),
+        "mn".to_string(),
+        "mr".to_string(),
+        "ms".to_string(),
+        "mt".to_string(),
+        "ne".to_string(),
+        "new".to_string(),
+        "nl".to_string(),
+        "no".to_string(),
+        "oc".to_string(),
+        "pi".to_string(),
+        "pl".to_string(),
+        "pt".to_string(),
+        "ro".to_string(),
+        "ru".to_string(),
+        "rs_cyrillic".to_string(),
+        "rs_latin".to_string(),
+        "sck".to_string(),
+        "sk".to_string(),
+        "sl".to_string(),
+        "sq".to_string(),
+        "sv".to_string(),
+        "sw".to_string(),
+        "ta".to_string(),
+        "tab".to_string(),
+        "te".to_string(),
+        "th".to_string(),
+        "tjk".to_string(),
+        "tl".to_string(),
+        "tr".to_string(),
+        "ug".to_string(),
+        "uk".to_string(),
+        "ur".to_string(),
+        "uz".to_string(),
+        "vi".to_string(),
+    ]
+}

data/vendor/kreuzberg/src/ocr/backends/mod.rs

@@ -0,0 +1,7 @@
+//! OCR backend language support modules.
+//!
+//! Each module provides language lists for a specific OCR backend.
+
+pub(super) mod easyocr;
+pub(super) mod paddleocr;
+pub(super) mod tesseract;

data/vendor/kreuzberg/src/ocr/backends/paddleocr.rs

@@ -0,0 +1,27 @@
+//! PaddleOCR language support.
+//!
+//! PaddleOCR supports 14 optimized languages for production deployments.
+
+/// Get list of languages supported by PaddleOCR.
+///
+/// # Returns
+///
+/// A vector of 14 language codes supported by PaddleOCR.
+pub(in crate::ocr) fn languages() -> Vec<String> {
+    vec![
+        "ch".to_string(),
+        "en".to_string(),
+        "french".to_string(),
+        "german".to_string(),
+        "korean".to_string(),
+        "japan".to_string(),
+        "chinese_cht".to_string(),
+        "ta".to_string(),
+        "te".to_string(),
+        "ka".to_string(),
+        "latin".to_string(),
+        "arabic".to_string(),
+        "cyrillic".to_string(),
+        "devanagari".to_string(),
+    ]
+}

data/vendor/kreuzberg/src/ocr/backends/tesseract.rs

@@ -0,0 +1,134 @@
+//! Tesseract OCR language support.
+//!
+//! Tesseract supports 100+ languages via the Tesseract OCR engine.
+
+/// Get list of languages supported by Tesseract OCR.
+///
+/// # Returns
+///
+/// A vector of 100+ language codes supported by Tesseract.
+pub(in crate::ocr) fn languages() -> Vec<String> {
+    vec![
+        "afr".to_string(),
+        "amh".to_string(),
+        "ara".to_string(),
+        "asm".to_string(),
+        "aze".to_string(),
+        "aze_cyrl".to_string(),
+        "bel".to_string(),
+        "ben".to_string(),
+        "bod".to_string(),
+        "bos".to_string(),
+        "bre".to_string(),
+        "bul".to_string(),
+        "cat".to_string(),
+        "ceb".to_string(),
+        "ces".to_string(),
+        "chi_sim".to_string(),
+        "chi_tra".to_string(),
+        "chr".to_string(),
+        "cos".to_string(),
+        "cym".to_string(),
+        "dan".to_string(),
+        "deu".to_string(),
+        "div".to_string(),
+        "dzo".to_string(),
+        "ell".to_string(),
+        "eng".to_string(),
+        "enm".to_string(),
+        "epo".to_string(),
+        "equ".to_string(),
+        "est".to_string(),
+        "eus".to_string(),
+        "fao".to_string(),
+        "fas".to_string(),
+        "fil".to_string(),
+        "fin".to_string(),
+        "fra".to_string(),
+        "frk".to_string(),
+        "frm".to_string(),
+        "fry".to_string(),
+        "gla".to_string(),
+        "gle".to_string(),
+        "glg".to_string(),
+        "grc".to_string(),
+        "guj".to_string(),
+        "hat".to_string(),
+        "heb".to_string(),
+        "hin".to_string(),
+        "hrv".to_string(),
+        "hun".to_string(),
+        "hye".to_string(),
+        "iku".to_string(),
+        "ind".to_string(),
+        "isl".to_string(),
+        "ita".to_string(),
+        "ita_old".to_string(),
+        "jav".to_string(),
+        "jpn".to_string(),
+        "kan".to_string(),
+        "kat".to_string(),
+        "kat_old".to_string(),
+        "kaz".to_string(),
+        "khm".to_string(),
+        "kir".to_string(),
+        "kmr".to_string(),
+        "kor".to_string(),
+        "lao".to_string(),
+        "lat".to_string(),
+        "lav".to_string(),
+        "lit".to_string(),
+        "ltz".to_string(),
+        "mal".to_string(),
+        "mar".to_string(),
+        "mkd".to_string(),
+        "mlt".to_string(),
+        "mon".to_string(),
+        "mri".to_string(),
+        "msa".to_string(),
+        "mya".to_string(),
+        "nep".to_string(),
+        "nld".to_string(),
+        "nor".to_string(),
+        "oci".to_string(),
+        "ori".to_string(),
+        "osd".to_string(),
+        "pan".to_string(),
+        "pol".to_string(),
+        "por".to_string(),
+        "pus".to_string(),
+        "que".to_string(),
+        "ron".to_string(),
+        "rus".to_string(),
+        "san".to_string(),
+        "sin".to_string(),
+        "slk".to_string(),
+        "slv".to_string(),
+        "snd".to_string(),
+        "spa".to_string(),
+        "spa_old".to_string(),
+        "sqi".to_string(),
+        "srp".to_string(),
+        "srp_latn".to_string(),
+        "sun".to_string(),
+        "swa".to_string(),
+        "swe".to_string(),
+        "syr".to_string(),
+        "tam".to_string(),
+        "tat".to_string(),
+        "tel".to_string(),
+        "tgk".to_string(),
+        "tha".to_string(),
+        "tir".to_string(),
+        "ton".to_string(),
+        "tur".to_string(),
+        "uig".to_string(),
+        "ukr".to_string(),
+        "urd".to_string(),
+        "uzb".to_string(),
+        "uzb_cyrl".to_string(),
+        "vie".to_string(),
+        "yid".to_string(),
+        "yor".to_string(),
+    ]
+}

data/vendor/kreuzberg/src/ocr/hocr.rs

@@ -1,7 +1,15 @@
 use super::error::OcrError;
-use html_to_markdown_rs::{ConversionOptions, convert};
+use crate::core::config::OutputFormat as KreuzbergOutputFormat;
+use html_to_markdown_rs::{ConversionOptions, OutputFormat as LibOutputFormat, convert};
 
-pub fn convert_hocr_to_markdown(hocr_html: &str, options: Option<ConversionOptions>) -> Result<String, OcrError> {
+/// Convert hOCR to specified output format (markdown or djot).
+///
+/// Defaults to Markdown for backward compatibility.
+pub fn convert_hocr_to_markdown(
+    hocr_html: &str,
+    options: Option<ConversionOptions>,
+    output_format: Option<KreuzbergOutputFormat>,
+) -> Result<String, OcrError> {
     let use_default = options.is_none();
     let mut opts = options.unwrap_or_default();
 
@@ -10,6 +18,14 @@ pub fn convert_hocr_to_markdown(hocr_html: &str, options: Option<ConversionOptio
         opts.extract_metadata = false;
     }
 
+    // Set output format
+    let format = output_format.unwrap_or(KreuzbergOutputFormat::Markdown);
+    opts.output_format = match format {
+        KreuzbergOutputFormat::Markdown => LibOutputFormat::Markdown,
+        KreuzbergOutputFormat::Djot => LibOutputFormat::Djot,
+        KreuzbergOutputFormat::Plain | KreuzbergOutputFormat::Html => LibOutputFormat::Markdown,
+    };
+
     convert(hocr_html, Some(opts)).map_err(|e| OcrError::ProcessingFailed(format!("hOCR conversion failed: {}", e)))
 }
 
@@ -26,7 +42,7 @@ mod tests {
             </p>
         </div>"#;
 
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(markdown.contains("Hello"));
         assert!(markdown.contains("World"));
     }
@@ -40,14 +56,14 @@ mod tests {
             </p>
         </div>"#;
 
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(!markdown.is_empty());
     }
 
     #[test]
     fn test_empty_hocr() {
         let hocr = "";
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(markdown.is_empty() || markdown.trim().is_empty());
     }
 
@@ -60,7 +76,7 @@ mod tests {
             </p>
         </div>"#;
 
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(!markdown.is_empty());
         assert!(markdown.contains("Content"));
     }
@@ -78,7 +94,7 @@ mod tests {
             </p>
         </div>"#;
 
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(markdown.contains("First"));
         assert!(markdown.contains("Second"));
     }
@@ -98,7 +114,7 @@ mod tests {
             </p>
         </div>"#;
 
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(!markdown.is_empty());
     }
 
@@ -111,7 +127,7 @@ mod tests {
             </p>
         </div>"#;
 
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(!markdown.is_empty());
     }
 
@@ -124,7 +140,7 @@ mod tests {
             </p>
         </div>"#;
 
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(!markdown.is_empty());
     }
 
@@ -140,7 +156,7 @@ mod tests {
             </div>
         </div>"#;
 
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(markdown.contains("Nested"));
     }
 
@@ -151,7 +167,7 @@ mod tests {
                 <span class="ocrx_word">Unclosed
         </div>"#;
 
-        let result = convert_hocr_to_markdown(hocr, None);
+        let result = convert_hocr_to_markdown(hocr, None, None);
         assert!(result.is_ok());
     }
 
@@ -163,7 +179,7 @@ mod tests {
             </p>
         </div>"#;
 
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(!markdown.is_empty());
     }
 
@@ -179,7 +195,7 @@ mod tests {
             </ul>
         </div>"#;
 
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(markdown.contains("Heading") || markdown.contains("heading") || !markdown.is_empty());
     }
 
@@ -193,7 +209,7 @@ mod tests {
             </p>
         </div>"#;
 
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        let markdown = convert_hocr_to_markdown(hocr, None, None).unwrap();
         assert!(markdown.contains("Ñoño") || !markdown.is_empty());
     }
 
@@ -208,9 +224,37 @@ mod tests {
         }
         hocr.push_str("</div>");
 
-        let result = convert_hocr_to_markdown(&hocr, None);
+        let result = convert_hocr_to_markdown(&hocr, None, None);
         assert!(result.is_ok());
         let markdown = result.unwrap();
         assert!(!markdown.is_empty());
     }
+
+    #[test]
+    fn test_hocr_to_djot_conversion() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">Hello</span>
+                <span class="ocrx_word">World</span>
+            </p>
+        </div>"#;
+
+        let result = convert_hocr_to_markdown(hocr, None, Some(KreuzbergOutputFormat::Djot)).unwrap();
+        assert!(result.contains("Hello"));
+        assert!(result.contains("World"));
+    }
+
+    #[test]
+    fn test_hocr_to_djot_with_formatting() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <strong class="ocrx_word">Bold</strong>
+                <em class="ocrx_word">Italic</em>
+            </p>
+        </div>"#;
+
+        let result = convert_hocr_to_markdown(hocr, None, Some(KreuzbergOutputFormat::Djot)).unwrap();
+        // Djot uses * for strong, _ for emphasis
+        assert!(!result.is_empty());
+    }
 }