kreuzberg 4.0.0.pre.rc.6

data/vendor/kreuzberg/src/extractors/xml.rs

@@ -0,0 +1,135 @@
+//! XML extractor.
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::extraction::xml::parse_xml;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::ExtractionResult;
+use async_trait::async_trait;
+
+/// XML extractor.
+///
+/// Extracts text content from XML files, preserving element structure information.
+pub struct XmlExtractor;
+
+impl XmlExtractor {
+    /// Create a new XML extractor.
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for XmlExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Plugin for XmlExtractor {
+    fn name(&self) -> &str {
+        "xml-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn description(&self) -> &str {
+        "Extracts text content from XML files with element metadata"
+    }
+
+    fn author(&self) -> &str {
+        "Kreuzberg Team"
+    }
+}
+
+#[async_trait]
+impl DocumentExtractor for XmlExtractor {
+    #[cfg_attr(feature = "otel", tracing::instrument(
+        skip(self, content, _config),
+        fields(
+            extractor.name = self.name(),
+            content.size_bytes = content.len(),
+        )
+    ))]
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let xml_result = parse_xml(content, false)?;
+
+        Ok(ExtractionResult {
+            content: xml_result.content,
+            mime_type: mime_type.to_string(),
+            metadata: crate::types::Metadata {
+                format: Some(crate::types::FormatMetadata::Xml(crate::types::XmlMetadata {
+                    element_count: xml_result.element_count,
+                    unique_elements: xml_result.unique_elements,
+                })),
+                ..Default::default()
+            },
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &["application/xml", "text/xml", "image/svg+xml"]
+    }
+
+    fn priority(&self) -> i32 {
+        50
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_xml_extractor() {
+        let extractor = XmlExtractor::new();
+        let content = b"<root><item>Hello</item><item>World</item></root>";
+        let config = ExtractionConfig::default();
+
+        let result = extractor
+            .extract_bytes(content, "application/xml", &config)
+            .await
+            .unwrap();
+
+        assert_eq!(result.mime_type, "application/xml");
+        assert_eq!(result.content, "Hello World");
+        assert!(result.metadata.format.is_some());
+        let xml_meta = match result.metadata.format.as_ref().unwrap() {
+            crate::types::FormatMetadata::Xml(meta) => meta,
+            _ => panic!("Expected Xml metadata"),
+        };
+        assert_eq!(xml_meta.element_count, 3);
+        assert!(xml_meta.unique_elements.contains(&"root".to_string()));
+        assert!(xml_meta.unique_elements.contains(&"item".to_string()));
+    }
+
+    #[test]
+    fn test_xml_plugin_interface() {
+        let extractor = XmlExtractor::new();
+        assert_eq!(extractor.name(), "xml-extractor");
+        assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
+        assert_eq!(
+            extractor.supported_mime_types(),
+            &["application/xml", "text/xml", "image/svg+xml"]
+        );
+        assert_eq!(extractor.priority(), 50);
+    }
+}

data/vendor/kreuzberg/src/image/dpi.rs

@@ -0,0 +1,164 @@
+/// PDF points per inch constant
+const PDF_POINTS_PER_INCH: f64 = 72.0;
+
+/// Calculate smart DPI based on page dimensions, memory constraints, and target DPI
+#[allow(clippy::cast_possible_truncation)]
+pub fn calculate_smart_dpi(
+    page_width: f64,
+    page_height: f64,
+    target_dpi: i32,
+    max_dimension: i32,
+    max_memory_mb: f64,
+) -> i32 {
+    let width_inches = page_width / PDF_POINTS_PER_INCH;
+    let height_inches = page_height / PDF_POINTS_PER_INCH;
+
+    let max_pixels = (max_memory_mb * 1024.0 * 1024.0 / 3.0).sqrt().round() as i32;
+
+    let max_dpi_for_memory_width = if width_inches > 0.0 {
+        (f64::from(max_pixels) / width_inches).round() as i32
+    } else {
+        target_dpi
+    };
+
+    let max_dpi_for_memory_height = if height_inches > 0.0 {
+        (f64::from(max_pixels) / height_inches).round() as i32
+    } else {
+        target_dpi
+    };
+
+    let memory_constrained_dpi = max_dpi_for_memory_width.min(max_dpi_for_memory_height);
+
+    let dimension_constrained_dpi =
+        calculate_dimension_constrained_dpi(width_inches, height_inches, target_dpi, max_dimension);
+
+    let final_dpi = target_dpi.min(memory_constrained_dpi).min(dimension_constrained_dpi);
+
+    final_dpi.max(72)
+}
+
+/// Calculate DPI constrained by maximum dimension
+#[allow(clippy::cast_possible_truncation)]
+fn calculate_dimension_constrained_dpi(
+    width_inches: f64,
+    height_inches: f64,
+    target_dpi: i32,
+    max_dimension: i32,
+) -> i32 {
+    let target_width_pixels = (width_inches * f64::from(target_dpi)).round() as i32;
+    let target_height_pixels = (height_inches * f64::from(target_dpi)).round() as i32;
+    let max_pixel_dimension = target_width_pixels.max(target_height_pixels);
+
+    if max_pixel_dimension > max_dimension {
+        let max_dpi_for_width = if width_inches > 0.0 {
+            (f64::from(max_dimension) / width_inches).round() as i32
+        } else {
+            target_dpi
+        };
+
+        let max_dpi_for_height = if height_inches > 0.0 {
+            (f64::from(max_dimension) / height_inches).round() as i32
+        } else {
+            target_dpi
+        };
+
+        max_dpi_for_width.min(max_dpi_for_height)
+    } else {
+        target_dpi
+    }
+}
+
+/// Calculate optimal DPI with min/max constraints
+pub fn calculate_optimal_dpi(
+    page_width: f64,
+    page_height: f64,
+    target_dpi: i32,
+    max_dimension: i32,
+    min_dpi: i32,
+    max_dpi: i32,
+) -> i32 {
+    let smart_dpi = calculate_smart_dpi(page_width, page_height, target_dpi, max_dimension, 2048.0);
+
+    min_dpi.max(smart_dpi.min(max_dpi))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_calculate_smart_dpi_basic() {
+        let dpi = calculate_smart_dpi(612.0, 792.0, 300, 4096, 2048.0);
+        assert!(dpi >= 72);
+        assert!(dpi <= 300);
+    }
+
+    #[test]
+    fn test_calculate_smart_dpi_memory_constrained() {
+        let dpi = calculate_smart_dpi(1224.0, 1584.0, 300, 8192, 10.0);
+        assert!(dpi < 300);
+        assert!(dpi >= 72);
+    }
+
+    #[test]
+    fn test_calculate_smart_dpi_dimension_constrained() {
+        let dpi = calculate_smart_dpi(612.0, 792.0, 300, 1000, 2048.0);
+        assert!(dpi < 300);
+    }
+
+    #[test]
+    fn test_calculate_smart_dpi_minimum_dpi() {
+        let dpi = calculate_smart_dpi(10000.0, 10000.0, 300, 100, 1.0);
+        assert_eq!(dpi, 72);
+    }
+
+    #[test]
+    fn test_calculate_smart_dpi_zero_dimensions() {
+        let dpi = calculate_smart_dpi(0.0, 792.0, 300, 4096, 2048.0);
+        assert!(dpi >= 72);
+
+        let dpi = calculate_smart_dpi(612.0, 0.0, 300, 4096, 2048.0);
+        assert!(dpi >= 72);
+
+        let dpi = calculate_smart_dpi(0.0, 0.0, 300, 4096, 2048.0);
+        assert_eq!(dpi, 300);
+    }
+
+    #[test]
+    fn test_calculate_dimension_constrained_dpi() {
+        let dpi = calculate_dimension_constrained_dpi(8.5, 11.0, 300, 4096);
+        assert!(dpi <= 300);
+
+        let dpi = calculate_dimension_constrained_dpi(8.5, 11.0, 600, 2000);
+        assert!(dpi < 600);
+    }
+
+    #[test]
+    fn test_calculate_optimal_dpi() {
+        let dpi = calculate_optimal_dpi(612.0, 792.0, 300, 4096, 72, 600);
+        assert!(dpi >= 72);
+        assert!(dpi <= 600);
+
+        let dpi = calculate_optimal_dpi(10000.0, 10000.0, 300, 100, 100, 600);
+        assert_eq!(dpi, 100);
+
+        let dpi = calculate_optimal_dpi(72.0, 72.0, 1000, 10000, 72, 600);
+        assert_eq!(dpi, 600);
+    }
+
+    #[test]
+    fn test_memory_calculation() {
+        let dpi = calculate_smart_dpi(612.0, 792.0, 10000, 100000, 2048.0);
+        assert!(dpi < 10000);
+        assert!(dpi >= 72);
+    }
+
+    #[test]
+    fn test_aspect_ratio_preservation() {
+        let wide_dpi = calculate_smart_dpi(1224.0, 396.0, 300, 4096, 2048.0);
+        let tall_dpi = calculate_smart_dpi(396.0, 1224.0, 300, 4096, 2048.0);
+
+        assert!(wide_dpi >= 72);
+        assert!(tall_dpi >= 72);
+    }
+}

data/vendor/kreuzberg/src/image/mod.rs

@@ -0,0 +1,6 @@
+pub mod dpi;
+pub mod preprocessing;
+pub mod resize;
+
+pub use dpi::calculate_optimal_dpi;
+pub use preprocessing::{NormalizeResult, normalize_image_dpi};