kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/extraction/html/stack_management.rs

@@ -0,0 +1,103 @@
+//! Stack management for HTML extraction with support for large documents.
+//!
+//! This module handles the specialized concern of managing stack size for HTML conversion,
+//! particularly for large HTML documents that may require more stack space than the default.
+//! On WASM, stack size is limited and cannot be increased, so size limits are enforced.
+//! On native platforms, dedicated threads with larger stacks are used for large HTML.
+
+use crate::error::{KreuzbergError, Result};
+
+#[cfg(not(target_arch = "wasm32"))]
+use std::{any::Any, thread};
+
+#[cfg(target_arch = "wasm32")]
+pub const MAX_HTML_SIZE_BYTES: usize = 2 * 1024 * 1024;
+
+#[cfg(not(target_arch = "wasm32"))]
+pub const LARGE_HTML_STACK_THRESHOLD_BYTES: usize = 512 * 1024;
+
+#[cfg(not(target_arch = "wasm32"))]
+pub const HTML_CONVERSION_STACK_SIZE_BYTES: usize = 16 * 1024 * 1024;
+
+/// Check if HTML size exceeds WASM limit and return error if so.
+///
+/// WASM builds have a fixed stack size that cannot be increased, so we enforce
+/// a 2MB limit to prevent stack overflow during HTML conversion.
+#[cfg(target_arch = "wasm32")]
+pub fn check_wasm_size_limit(html: &str) -> Result<()> {
+    if html.len() > MAX_HTML_SIZE_BYTES {
+        return Err(KreuzbergError::validation(format!(
+            "HTML file size ({} bytes) exceeds WASM limit of {} bytes (2MB). \
+             Large HTML files cannot be processed in WASM due to stack constraints. \
+             Consider using the native library for files of this size.",
+            html.len(),
+            MAX_HTML_SIZE_BYTES
+        )));
+    }
+    Ok(())
+}
+
+/// Check if HTML size exceeds WASM limit and return error if so.
+///
+/// No-op on non-WASM platforms.
+#[cfg(not(target_arch = "wasm32"))]
+pub fn check_wasm_size_limit(_html: &str) -> Result<()> {
+    Ok(())
+}
+
+/// Determine if HTML requires a dedicated stack due to size.
+///
+/// On native platforms, HTML larger than the threshold will be processed
+/// on a dedicated thread with a larger stack to prevent overflow.
+#[cfg(not(target_arch = "wasm32"))]
+pub fn html_requires_large_stack(len: usize) -> bool {
+    len >= LARGE_HTML_STACK_THRESHOLD_BYTES
+}
+
+/// Run a job on a dedicated thread with a large stack.
+///
+/// This is useful for HTML conversion of large documents that might
+/// overflow the default thread stack on native platforms.
+///
+/// # Arguments
+///
+/// * `job` - The closure to execute on the dedicated thread
+///
+/// # Returns
+///
+/// The result of the job execution, or an error if the thread panics
+#[cfg(not(target_arch = "wasm32"))]
+pub fn run_on_dedicated_stack<T, F>(job: F) -> Result<T>
+where
+    T: Send + 'static,
+    F: FnOnce() -> Result<T> + Send + 'static,
+{
+    let handle = thread::Builder::new()
+        .name("kreuzberg-html-conversion".to_string())
+        .stack_size(HTML_CONVERSION_STACK_SIZE_BYTES)
+        .spawn(job)
+        .map_err(|err| KreuzbergError::Other(format!("Failed to spawn HTML conversion thread: {}", err)))?;
+
+    match handle.join() {
+        Ok(result) => result,
+        Err(panic) => {
+            let reason = extract_panic_reason(&panic);
+            Err(KreuzbergError::Other(format!("HTML conversion panicked: {}", reason)))
+        }
+    }
+}
+
+/// Extract a human-readable reason from a panic.
+///
+/// Attempts to downcast the panic value to either &str or String,
+/// falling back to a generic message if neither succeeds.
+#[cfg(not(target_arch = "wasm32"))]
+fn extract_panic_reason(panic: &Box<dyn Any + Send + 'static>) -> String {
+    if let Some(msg) = panic.downcast_ref::<&str>() {
+        (*msg).to_string()
+    } else if let Some(msg) = panic.downcast_ref::<String>() {
+        msg.clone()
+    } else {
+        "unknown panic".to_string()
+    }
+}

data/vendor/kreuzberg/src/extraction/html/types.rs

@@ -0,0 +1,28 @@
+//! Type definitions for HTML extraction.
+
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+pub use html_to_markdown_rs::{
+    CodeBlockStyle, HeadingStyle, HighlightStyle, ListIndentType, NewlineStyle, PreprocessingOptions,
+    PreprocessingPreset, WhitespaceMode,
+};
+
+/// Result of HTML extraction with optional images and warnings.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HtmlExtractionResult {
+    pub markdown: String,
+    pub images: Vec<ExtractedInlineImage>,
+    pub warnings: Vec<String>,
+}
+
+/// Extracted inline image with metadata.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ExtractedInlineImage {
+    pub data: Vec<u8>,
+    pub format: String,
+    pub filename: Option<String>,
+    pub description: Option<String>,
+    pub dimensions: Option<(u32, u32)>,
+    pub attributes: HashMap<String, String>,
+}

data/vendor/kreuzberg/src/extraction/mod.rs

@@ -1,5 +1,6 @@
 pub mod structured;
 pub mod text;
+pub mod transform;
 
 #[cfg(feature = "ocr")]
 pub mod image;
@@ -41,11 +42,14 @@ pub mod table;
 #[cfg(feature = "xml")]
 pub mod xml;
 
-#[cfg(any(feature = "office", feature = "html"))]
+#[cfg(any(feature = "office", feature = "html", feature = "xml"))]
 pub mod markdown;
 
 pub use structured::{JsonExtractionConfig, StructuredDataResult, parse_json, parse_toml, parse_yaml};
 pub use text::parse_text;
+pub use transform::{
+    ListItemMetadata, ListType, detect_list_items, generate_element_id, transform_extraction_result_to_elements,
+};
 
 #[cfg(feature = "ocr")]
 pub use image::{ImageMetadata, extract_image_metadata};
@@ -84,7 +88,7 @@ pub use table::table_from_arrow_to_markdown;
 #[cfg(feature = "xml")]
 pub use xml::parse_xml;
 
-#[cfg(any(feature = "office", feature = "html"))]
+#[cfg(any(feature = "office", feature = "html", feature = "xml"))]
 pub use markdown::cells_to_markdown;
 
 pub use capacity::{

data/vendor/kreuzberg/src/extraction/pptx/container.rs

@@ -0,0 +1,159 @@
+//! PPTX container and ZIP archive management.
+//!
+//! This module handles opening PPTX files, reading files from the ZIP archive,
+//! finding slide paths, and iterating through slides.
+
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::Read;
+use std::path::Path;
+use zip::ZipArchive;
+
+use super::elements::Slide;
+use super::image_handling::get_full_image_path;
+use crate::error::{KreuzbergError, Result};
+
+pub(super) struct PptxContainer {
+    pub(super) archive: ZipArchive<File>,
+    slide_paths: Vec<String>,
+}
+
+impl PptxContainer {
+    pub(super) fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+        // IO errors must bubble up unchanged - file access issues need user reports ~keep
+        let file = File::open(path)?;
+
+        let mut archive = match ZipArchive::new(file) {
+            Ok(arc) => arc,
+            Err(zip::result::ZipError::Io(io_err)) => return Err(io_err.into()), // Bubble up IO errors ~keep
+            Err(e) => {
+                return Err(KreuzbergError::parsing(format!(
+                    "Failed to read PPTX archive (invalid format): {}",
+                    e
+                )));
+            }
+        };
+
+        let slide_paths = Self::find_slide_paths(&mut archive)?;
+
+        Ok(Self { archive, slide_paths })
+    }
+
+    pub(super) fn slide_paths(&self) -> &[String] {
+        &self.slide_paths
+    }
+
+    pub(super) fn read_file(&mut self, path: &str) -> Result<Vec<u8>> {
+        match self.archive.by_name(path) {
+            Ok(mut file) => {
+                let mut contents = Vec::new();
+                // IO errors must bubble up - file read issues need user reports ~keep
+                file.read_to_end(&mut contents)?;
+                Ok(contents)
+            }
+            Err(zip::result::ZipError::FileNotFound) => {
+                Err(KreuzbergError::parsing("File not found in archive".to_string()))
+            }
+            Err(zip::result::ZipError::Io(io_err)) => Err(io_err.into()), // Bubble up IO errors ~keep
+            Err(e) => Err(KreuzbergError::parsing(format!("Zip error: {}", e))),
+        }
+    }
+
+    pub(super) fn get_slide_rels_path(&self, slide_path: &str) -> String {
+        super::image_handling::get_slide_rels_path(slide_path)
+    }
+
+    fn find_slide_paths(archive: &mut ZipArchive<File>) -> Result<Vec<String>> {
+        if let Ok(rels_data) = Self::read_file_from_archive(archive, "ppt/_rels/presentation.xml.rels")
+            && let Ok(paths) = super::parser::parse_presentation_rels(&rels_data)
+        {
+            return Ok(paths);
+        }
+
+        let mut slide_paths = Vec::new();
+        for i in 0..archive.len() {
+            if let Ok(file) = archive.by_index(i) {
+                let name = file.name();
+                if name.starts_with("ppt/slides/slide") && name.ends_with(".xml") {
+                    slide_paths.push(name.to_string());
+                }
+            }
+        }
+
+        slide_paths.sort();
+        Ok(slide_paths)
+    }
+
+    fn read_file_from_archive(archive: &mut ZipArchive<File>, path: &str) -> Result<Vec<u8>> {
+        let mut file = match archive.by_name(path) {
+            Ok(f) => f,
+            Err(zip::result::ZipError::Io(io_err)) => return Err(io_err.into()), // Bubble up IO errors ~keep
+            Err(e) => {
+                return Err(KreuzbergError::parsing(format!(
+                    "Failed to read file from archive: {}",
+                    e
+                )));
+            }
+        };
+        let mut contents = Vec::new();
+        // IO errors must bubble up - file read issues need user reports ~keep
+        file.read_to_end(&mut contents)?;
+        Ok(contents)
+    }
+}
+
+pub(super) struct SlideIterator {
+    container: PptxContainer,
+    current_index: usize,
+    total_slides: usize,
+}
+
+impl SlideIterator {
+    pub(super) fn new(container: PptxContainer) -> Self {
+        let total_slides = container.slide_paths().len();
+        Self {
+            container,
+            current_index: 0,
+            total_slides,
+        }
+    }
+
+    pub(super) fn slide_count(&self) -> usize {
+        self.total_slides
+    }
+
+    pub(super) fn next_slide(&mut self) -> Result<Option<Slide>> {
+        if self.current_index >= self.total_slides {
+            return Ok(None);
+        }
+
+        let slide_path = &self.container.slide_paths()[self.current_index].clone();
+        let slide_number = (self.current_index + 1) as u32;
+
+        let xml_data = self.container.read_file(slide_path)?;
+
+        let rels_path = self.container.get_slide_rels_path(slide_path);
+        let rels_data = self.container.read_file(&rels_path).ok();
+
+        let slide = Slide::from_xml(slide_number, &xml_data, rels_data.as_deref())?;
+
+        self.current_index += 1;
+
+        Ok(Some(slide))
+    }
+
+    pub(super) fn get_slide_images(&mut self, slide: &Slide) -> Result<HashMap<String, Vec<u8>>> {
+        let mut image_data = HashMap::new();
+
+        for img_ref in &slide.images {
+            let slide_path = &self.container.slide_paths()[slide.slide_number as usize - 1];
+            let full_path = get_full_image_path(slide_path, &img_ref.target);
+
+            if let Ok(data) = self.container.read_file(&full_path) {
+                image_data.insert(img_ref.id.clone(), data);
+            }
+        }
+
+        Ok(image_data)
+    }
+}

data/vendor/kreuzberg/src/extraction/pptx/content_builder.rs

@@ -0,0 +1,168 @@
+//! Content builder for accumulating slide output.
+//!
+//! This module provides utilities for building the final markdown content
+//! from slide elements and managing page boundaries.
+
+pub(super) struct ContentBuilder {
+    pub(super) content: String,
+    pub(super) boundaries: Vec<crate::types::PageBoundary>,
+    pub(super) page_contents: Vec<crate::types::PageContent>,
+    pub(super) config: Option<crate::core::config::PageConfig>,
+}
+
+impl ContentBuilder {
+    pub(super) fn new() -> Self {
+        Self {
+            content: String::with_capacity(8192),
+            boundaries: Vec::new(),
+            page_contents: Vec::new(),
+            config: None,
+        }
+    }
+
+    pub(super) fn with_page_config(capacity: usize, config: Option<crate::core::config::PageConfig>) -> Self {
+        Self {
+            content: String::with_capacity(capacity),
+            boundaries: if config.is_some() {
+                Vec::new()
+            } else {
+                Vec::with_capacity(0)
+            },
+            page_contents: if config.is_some() {
+                Vec::new()
+            } else {
+                Vec::with_capacity(0)
+            },
+            config,
+        }
+    }
+
+    pub(super) fn start_slide(&mut self, slide_number: u32) -> usize {
+        let byte_start = self.content.len();
+
+        if let Some(ref cfg) = self.config
+            && cfg.insert_page_markers
+        {
+            let marker = cfg.marker_format.replace("{page_num}", &slide_number.to_string());
+            self.content.push_str(&marker);
+        }
+
+        byte_start
+    }
+
+    pub(super) fn end_slide(&mut self, slide_number: u32, byte_start: usize, slide_content: String) {
+        let byte_end = self.content.len();
+
+        if self.config.is_some() {
+            self.boundaries.push(crate::types::PageBoundary {
+                byte_start,
+                byte_end,
+                page_number: slide_number as usize,
+            });
+
+            self.page_contents.push(crate::types::PageContent {
+                page_number: slide_number as usize,
+                content: slide_content,
+                tables: Vec::new(),
+                images: Vec::new(),
+                hierarchy: None,
+            });
+        }
+    }
+
+    pub(super) fn add_slide_header(&mut self, slide_number: u32) {
+        self.content.reserve(50);
+        self.content.push_str("\n\n<!-- Slide number: ");
+        self.content.push_str(&slide_number.to_string());
+        self.content.push_str(" -->\n");
+    }
+
+    pub(super) fn add_text(&mut self, text: &str) {
+        if !text.trim().is_empty() {
+            self.content.push_str(text);
+        }
+    }
+
+    pub(super) fn add_title(&mut self, title: &str) {
+        if !title.trim().is_empty() {
+            self.content.push_str("# ");
+            self.content.push_str(title.trim());
+            self.content.push('\n');
+        }
+    }
+
+    pub(super) fn add_table(&mut self, rows: &[Vec<String>]) {
+        if rows.is_empty() {
+            return;
+        }
+
+        self.content.push_str("\n<table>");
+        for (i, row) in rows.iter().enumerate() {
+            self.content.push_str("<tr>");
+            let tag = if i == 0 { "th" } else { "td" };
+
+            for cell in row {
+                self.content.push('<');
+                self.content.push_str(tag);
+                self.content.push('>');
+                self.content.push_str(&super::image_handling::html_escape(cell));
+                self.content.push_str("</");
+                self.content.push_str(tag);
+                self.content.push('>');
+            }
+            self.content.push_str("</tr>");
+        }
+        self.content.push_str("</table>\n");
+    }
+
+    pub(super) fn add_list_item(&mut self, level: u32, is_ordered: bool, text: &str) {
+        let indent_count = level.saturating_sub(1) as usize;
+        for _ in 0..indent_count {
+            self.content.push_str("  ");
+        }
+
+        let marker = if is_ordered { "1." } else { "-" };
+        self.content.push_str(marker);
+        self.content.push(' ');
+        self.content.push_str(text.trim());
+        self.content.push('\n');
+    }
+
+    pub(super) fn add_image(&mut self, image_id: &str, slide_number: u32) {
+        let filename = format!("slide_{}_image_{}.jpg", slide_number, image_id);
+        self.content.push_str("![");
+        self.content.push_str(image_id);
+        self.content.push_str("](");
+        self.content.push_str(&filename);
+        self.content.push_str(")\n");
+    }
+
+    pub(super) fn add_notes(&mut self, notes: &str) {
+        if !notes.trim().is_empty() {
+            self.content.push_str("\n\n### Notes:\n");
+            self.content.push_str(notes);
+            self.content.push('\n');
+        }
+    }
+
+    pub(super) fn build(
+        self,
+    ) -> (
+        String,
+        Option<Vec<crate::types::PageBoundary>>,
+        Option<Vec<crate::types::PageContent>>,
+    ) {
+        let content = self.content.trim().to_string();
+        let boundaries = if self.config.is_some() && !self.boundaries.is_empty() {
+            Some(self.boundaries)
+        } else {
+            None
+        };
+        let pages = if self.config.is_some() && !self.page_contents.is_empty() {
+            Some(self.page_contents)
+        } else {
+            None
+        };
+        (content, boundaries, pages)
+    }
+}

data/vendor/kreuzberg/src/extraction/pptx/elements.rs

@@ -0,0 +1,132 @@
+//! Internal types for PPTX extraction.
+//!
+//! This module defines the internal data structures used to represent
+//! slide elements, formatting, and text runs during XML parsing.
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)]
+pub(super) struct ElementPosition {
+    pub(super) x: i64,
+    pub(super) y: i64,
+}
+
+#[derive(Debug, Clone, Default)]
+pub(super) struct Formatting {
+    pub(super) bold: bool,
+    pub(super) italic: bool,
+    pub(super) underlined: bool,
+    pub(super) lang: String,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct Run {
+    pub(super) text: String,
+    pub(super) formatting: Formatting,
+}
+
+impl Run {
+    pub(super) fn extract(&self) -> String {
+        self.text.clone()
+    }
+
+    pub(super) fn render_as_md(&self) -> String {
+        let mut result = self.text.clone();
+
+        if self.formatting.bold {
+            result = format!("**{}**", result);
+        }
+        if self.formatting.italic {
+            result = format!("*{}*", result);
+        }
+        if self.formatting.underlined {
+            result = format!("<u>{}</u>", result);
+        }
+
+        result
+    }
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct TextElement {
+    pub(super) runs: Vec<Run>,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct ListItem {
+    pub(super) level: u32,
+    pub(super) is_ordered: bool,
+    pub(super) runs: Vec<Run>,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct ListElement {
+    pub(super) items: Vec<ListItem>,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct TableCell {
+    pub(super) runs: Vec<Run>,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct TableRow {
+    pub(super) cells: Vec<TableCell>,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct TableElement {
+    pub(super) rows: Vec<TableRow>,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct ImageReference {
+    pub(super) id: String,
+    pub(super) target: String,
+}
+
+#[derive(Debug, Clone)]
+pub(super) enum SlideElement {
+    Text(TextElement, ElementPosition),
+    Table(TableElement, ElementPosition),
+    Image(ImageReference, ElementPosition),
+    List(ListElement, ElementPosition),
+    Unknown,
+}
+
+impl SlideElement {
+    pub(super) fn position(&self) -> ElementPosition {
+        match self {
+            SlideElement::Text(_, pos)
+            | SlideElement::Table(_, pos)
+            | SlideElement::Image(_, pos)
+            | SlideElement::List(_, pos) => *pos,
+            SlideElement::Unknown => ElementPosition::default(),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub(super) struct Slide {
+    pub(super) slide_number: u32,
+    pub(super) elements: Vec<SlideElement>,
+    pub(super) images: Vec<ImageReference>,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct ParserConfig {
+    pub(super) extract_images: bool,
+    pub(super) include_slide_comment: bool,
+}
+
+impl Default for ParserConfig {
+    fn default() -> Self {
+        Self {
+            extract_images: true,
+            include_slide_comment: false,
+        }
+    }
+}
+
+pub(super) enum ParsedContent {
+    Text(TextElement),
+    List(ListElement),
+}

data/vendor/kreuzberg/src/extraction/pptx/image_handling.rs

@@ -0,0 +1,57 @@
+//! Image parsing and format detection.
+//!
+//! This module handles image-related parsing from slide XML and
+//! detection of image formats from file data.
+
+pub(super) fn html_escape(text: &str) -> String {
+    text.replace('&', "&")
+        .replace('<', "&lt;")
+        .replace('>', "&gt;")
+        .replace('"', "&quot;")
+        .replace('\'', "&#x27;")
+}
+
+pub(super) fn detect_image_format(data: &[u8]) -> String {
+    if data.starts_with(&[0xFF, 0xD8, 0xFF]) {
+        "jpeg".to_string()
+    } else if data.starts_with(&[0x89, 0x50, 0x4E, 0x47]) {
+        "png".to_string()
+    } else if data.starts_with(b"GIF") {
+        "gif".to_string()
+    } else if data.starts_with(b"BM") {
+        "bmp".to_string()
+    } else if data.starts_with(b"<svg") || data.starts_with(b"<?xml") {
+        "svg".to_string()
+    } else if data.starts_with(b"II\x2A\x00") || data.starts_with(b"MM\x00\x2A") {
+        "tiff".to_string()
+    } else {
+        "unknown".to_string()
+    }
+}
+
+pub(super) fn get_slide_rels_path(slide_path: &str) -> String {
+    let parts: Vec<&str> = slide_path.rsplitn(2, '/').collect();
+    if parts.len() == 2 {
+        format!("{}/_rels/{}.rels", parts[1], parts[0])
+    } else {
+        format!("_rels/{}.rels", slide_path)
+    }
+}
+
+pub(super) fn get_full_image_path(slide_path: &str, image_target: &str) -> String {
+    if image_target.starts_with("..") {
+        let parts: Vec<&str> = slide_path.rsplitn(3, '/').collect();
+        if parts.len() >= 3 {
+            format!("{}/{}", parts[2], &image_target[3..])
+        } else {
+            format!("ppt/{}", &image_target[3..])
+        }
+    } else {
+        let parts: Vec<&str> = slide_path.rsplitn(2, '/').collect();
+        if parts.len() == 2 {
+            format!("{}/{}", parts[1], image_target)
+        } else {
+            format!("ppt/slides/{}", image_target)
+        }
+    }
+}