kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/extractors/epub/content.rs

@@ -0,0 +1,333 @@
+//! EPUB content extraction and text processing.
+//!
+//! Handles extraction of text content from XHTML files in spine order,
+//! with markdown conversion and HTML cleaning utilities.
+
+use crate::Result;
+use std::io::Cursor;
+use zip::ZipArchive;
+
+use super::metadata::parse_opf;
+use super::parsing::{read_file_from_zip, resolve_path};
+
+/// Extract text content from an EPUB document by reading in spine order
+pub(super) fn extract_content(
+    archive: &mut ZipArchive<Cursor<Vec<u8>>>,
+    opf_path: &str,
+    manifest_dir: &str,
+) -> Result<String> {
+    let opf_xml = read_file_from_zip(archive, opf_path)?;
+    let (_, spine_hrefs) = parse_opf(&opf_xml)?;
+
+    let mut content = String::new();
+
+    for (index, href) in spine_hrefs.iter().enumerate() {
+        let file_path = resolve_path(manifest_dir, href);
+
+        match read_file_from_zip(archive, &file_path) {
+            Ok(xhtml_content) => {
+                let text = extract_text_from_xhtml(&xhtml_content);
+                if !text.is_empty() {
+                    if index > 0 && !content.ends_with('\n') {
+                        content.push('\n');
+                    }
+                    content.push_str(&text);
+                    content.push('\n');
+                }
+            }
+            Err(_) => {
+                continue;
+            }
+        }
+    }
+
+    Ok(content.trim().to_string())
+}
+
+/// Extract text from XHTML content using html-to-markdown-rs
+pub(super) fn extract_text_from_xhtml(xhtml: &str) -> String {
+    match crate::extraction::html::convert_html_to_markdown(xhtml, None, None) {
+        Ok(markdown) => {
+            let text = markdown_to_plain_text(&markdown);
+            remove_html_comments(&text)
+        }
+        Err(_) => strip_html_tags(xhtml),
+    }
+}
+
+/// Remove HTML comments from text
+pub(super) fn remove_html_comments(text: &str) -> String {
+    let mut result = String::new();
+    let mut in_comment = false;
+    let mut chars = text.chars().peekable();
+
+    while let Some(ch) = chars.next() {
+        if !in_comment && ch == '<' {
+            if chars.peek() == Some(&'!') {
+                chars.next();
+                if chars.peek() == Some(&'-') {
+                    chars.next();
+                    if chars.peek() == Some(&'-') {
+                        chars.next();
+                        in_comment = true;
+                        continue;
+                    } else {
+                        result.push('<');
+                        result.push('!');
+                        result.push('-');
+                        continue;
+                    }
+                } else {
+                    result.push('<');
+                    result.push('!');
+                    continue;
+                }
+            } else {
+                result.push(ch);
+            }
+        } else if in_comment {
+            if ch == '-' && chars.peek() == Some(&'-') {
+                chars.next();
+                if chars.peek() == Some(&'>') {
+                    chars.next();
+                    in_comment = false;
+                    result.push('\n');
+                }
+            }
+        } else {
+            result.push(ch);
+        }
+    }
+
+    result
+}
+
+/// Convert markdown output to plain text by removing markdown syntax
+pub(super) fn markdown_to_plain_text(markdown: &str) -> String {
+    let mut text = String::new();
+    let mut in_code_block = false;
+
+    for line in markdown.lines() {
+        let trimmed = line.trim();
+
+        if trimmed.is_empty() {
+            if !text.is_empty() && !text.ends_with('\n') {
+                text.push('\n');
+            }
+            continue;
+        }
+
+        if trimmed.starts_with("```") {
+            in_code_block = !in_code_block;
+            continue;
+        }
+
+        if in_code_block {
+            text.push_str(trimmed);
+            text.push('\n');
+            continue;
+        }
+
+        let cleaned = if let Some(stripped) = trimmed.strip_prefix("- ").or_else(|| trimmed.strip_prefix("* ")) {
+            stripped
+        } else if let Some(stripped) = trimmed.strip_prefix(|c: char| c.is_ascii_digit()) {
+            if let Some(rest) = stripped.strip_prefix(". ") {
+                rest
+            } else {
+                trimmed
+            }
+        } else {
+            trimmed
+        };
+
+        let cleaned = cleaned.trim_start_matches('#').trim();
+
+        let cleaned = cleaned
+            .replace("**", "")
+            .replace("__", "")
+            .replace("*", "")
+            .replace("_", "");
+
+        let cleaned = remove_markdown_links(&cleaned);
+
+        if !cleaned.is_empty() {
+            text.push_str(&cleaned);
+            text.push('\n');
+        }
+    }
+
+    text.trim().to_string()
+}
+
+/// Remove markdown links [text](url) -> text
+pub(super) fn remove_markdown_links(text: &str) -> String {
+    let mut result = String::new();
+    let mut chars = text.chars().peekable();
+
+    while let Some(ch) = chars.next() {
+        if ch == '[' {
+            let mut link_text = String::new();
+            let mut depth = 1;
+
+            while let Some(&next_ch) = chars.peek() {
+                chars.next();
+                if next_ch == '[' {
+                    depth += 1;
+                    link_text.push(next_ch);
+                } else if next_ch == ']' {
+                    depth -= 1;
+                    if depth == 0 {
+                        break;
+                    }
+                    link_text.push(next_ch);
+                } else {
+                    link_text.push(next_ch);
+                }
+            }
+
+            if let Some(&'(') = chars.peek() {
+                chars.next();
+                let mut paren_depth = 1;
+                while let Some(&next_ch) = chars.peek() {
+                    chars.next();
+                    if next_ch == '(' {
+                        paren_depth += 1;
+                    } else if next_ch == ')' {
+                        paren_depth -= 1;
+                        if paren_depth == 0 {
+                            break;
+                        }
+                    }
+                }
+            }
+
+            result.push_str(&link_text);
+        } else {
+            result.push(ch);
+        }
+    }
+
+    result
+}
+
+/// Fallback: strip HTML tags without using specialized libraries
+pub(super) fn strip_html_tags(html: &str) -> String {
+    let mut text = String::new();
+    let mut in_tag = false;
+    let mut in_script_style = false;
+    let mut tag_name = String::new();
+
+    for ch in html.chars() {
+        if ch == '<' {
+            in_tag = true;
+            tag_name.clear();
+            continue;
+        }
+
+        if ch == '>' {
+            in_tag = false;
+
+            let tag_lower = tag_name.to_lowercase();
+            if tag_lower.contains("script") || tag_lower.contains("style") {
+                in_script_style = !tag_name.starts_with('/');
+            }
+            continue;
+        }
+
+        if in_tag {
+            tag_name.push(ch);
+            continue;
+        }
+
+        if in_script_style {
+            continue;
+        }
+
+        if ch == '\n' || ch == '\r' || ch == '\t' || ch == ' ' {
+            if !text.is_empty() && !text.ends_with(' ') {
+                text.push(' ');
+            }
+        } else {
+            text.push(ch);
+        }
+    }
+
+    let mut result = String::new();
+    let mut prev_space = false;
+    for ch in text.chars() {
+        if ch == ' ' {
+            if !prev_space {
+                result.push(ch);
+            }
+            prev_space = true;
+        } else {
+            result.push(ch);
+            prev_space = false;
+        }
+    }
+
+    result.trim().to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_strip_html_tags_simple() {
+        let html = "<html><body><p>Hello World</p></body></html>";
+        let text = strip_html_tags(html);
+        assert!(text.contains("Hello World"));
+    }
+
+    #[test]
+    fn test_strip_html_tags_with_scripts() {
+        let html = "<body><p>Text</p><script>alert('bad');</script><p>More</p></body>";
+        let text = strip_html_tags(html);
+        assert!(!text.contains("bad"));
+        assert!(text.contains("Text"));
+        assert!(text.contains("More"));
+    }
+
+    #[test]
+    fn test_strip_html_tags_with_styles() {
+        let html = "<body><p>Text</p><style>.class { color: red; }</style><p>More</p></body>";
+        let text = strip_html_tags(html);
+        assert!(!text.to_lowercase().contains("color"));
+        assert!(text.contains("Text"));
+        assert!(text.contains("More"));
+    }
+
+    #[test]
+    fn test_strip_html_tags_normalizes_whitespace() {
+        let html = "<p>Hello   \n\t   World</p>";
+        let text = strip_html_tags(html);
+        assert!(text.contains("Hello") && text.contains("World"));
+    }
+
+    #[test]
+    fn test_remove_markdown_links() {
+        let text = "This is a [link](http://example.com) in text";
+        let result = remove_markdown_links(text);
+        assert!(result.contains("link"));
+        assert!(!result.contains("http://"));
+    }
+
+    #[test]
+    fn test_markdown_to_plain_text_removes_formatting() {
+        let markdown = "# Heading\n\nThis is **bold** text with _italic_ emphasis.";
+        let result = markdown_to_plain_text(markdown);
+        assert!(result.contains("Heading"));
+        assert!(result.contains("bold"));
+        assert!(!result.contains("**"));
+    }
+
+    #[test]
+    fn test_markdown_to_plain_text_removes_list_markers() {
+        let markdown = "- Item 1\n- Item 2\n* Item 3";
+        let result = markdown_to_plain_text(markdown);
+        assert!(result.contains("Item 1"));
+        assert!(result.contains("Item 2"));
+        assert!(result.contains("Item 3"));
+    }
+}

data/vendor/kreuzberg/src/extractors/epub/metadata.rs

@@ -0,0 +1,137 @@
+//! Metadata extraction from EPUB OPF files.
+//!
+//! Handles parsing of OPF (Open Packaging Format) files and extraction of
+//! Dublin Core metadata following EPUB2 and EPUB3 standards.
+
+use crate::Result;
+use roxmltree;
+use std::collections::BTreeMap;
+
+/// Metadata extracted from OPF (Open Packaging Format) file
+#[derive(Debug, Default, Clone)]
+pub(super) struct OepbMetadata {
+    pub(super) title: Option<String>,
+    pub(super) creator: Option<String>,
+    pub(super) date: Option<String>,
+    pub(super) language: Option<String>,
+    pub(super) identifier: Option<String>,
+    pub(super) publisher: Option<String>,
+    pub(super) subject: Option<String>,
+    pub(super) description: Option<String>,
+    pub(super) rights: Option<String>,
+}
+
+/// Extract metadata from EPUB OPF file
+pub(super) fn extract_metadata(opf_xml: &str) -> Result<(OepbMetadata, BTreeMap<String, serde_json::Value>)> {
+    let mut additional_metadata = BTreeMap::new();
+
+    let (epub_metadata, _) = parse_opf(opf_xml)?;
+
+    if let Some(identifier) = epub_metadata.identifier.clone() {
+        additional_metadata.insert("identifier".to_string(), serde_json::json!(identifier));
+    }
+
+    if let Some(publisher) = epub_metadata.publisher.clone() {
+        additional_metadata.insert("publisher".to_string(), serde_json::json!(publisher));
+    }
+
+    if let Some(subject) = epub_metadata.subject.clone() {
+        additional_metadata.insert("subject".to_string(), serde_json::json!(subject));
+    }
+
+    if let Some(description) = epub_metadata.description.clone() {
+        additional_metadata.insert("description".to_string(), serde_json::json!(description));
+    }
+
+    if let Some(rights) = epub_metadata.rights.clone() {
+        additional_metadata.insert("rights".to_string(), serde_json::json!(rights));
+    }
+
+    Ok((epub_metadata, additional_metadata))
+}
+
+/// Parse OPF file and extract metadata and spine order
+pub(super) fn parse_opf(xml: &str) -> Result<(OepbMetadata, Vec<String>)> {
+    match roxmltree::Document::parse(xml) {
+        Ok(doc) => {
+            let root = doc.root();
+
+            let mut metadata = OepbMetadata::default();
+            let mut manifest: BTreeMap<String, String> = BTreeMap::new();
+            let mut spine_order: Vec<String> = Vec::new();
+
+            for node in root.descendants() {
+                match node.tag_name().name() {
+                    "title" => {
+                        if let Some(text) = node.text() {
+                            metadata.title = Some(text.trim().to_string());
+                        }
+                    }
+                    "creator" => {
+                        if let Some(text) = node.text() {
+                            metadata.creator = Some(text.trim().to_string());
+                        }
+                    }
+                    "date" => {
+                        if let Some(text) = node.text() {
+                            metadata.date = Some(text.trim().to_string());
+                        }
+                    }
+                    "language" => {
+                        if let Some(text) = node.text() {
+                            metadata.language = Some(text.trim().to_string());
+                        }
+                    }
+                    "identifier" => {
+                        if let Some(text) = node.text() {
+                            metadata.identifier = Some(text.trim().to_string());
+                        }
+                    }
+                    "publisher" => {
+                        if let Some(text) = node.text() {
+                            metadata.publisher = Some(text.trim().to_string());
+                        }
+                    }
+                    "subject" => {
+                        if let Some(text) = node.text() {
+                            metadata.subject = Some(text.trim().to_string());
+                        }
+                    }
+                    "description" => {
+                        if let Some(text) = node.text() {
+                            metadata.description = Some(text.trim().to_string());
+                        }
+                    }
+                    "rights" => {
+                        if let Some(text) = node.text() {
+                            metadata.rights = Some(text.trim().to_string());
+                        }
+                    }
+                    "item" => {
+                        if let Some(id) = node.attribute("id")
+                            && let Some(href) = node.attribute("href")
+                        {
+                            manifest.insert(id.to_string(), href.to_string());
+                        }
+                    }
+                    _ => {}
+                }
+            }
+
+            for node in root.descendants() {
+                if node.tag_name().name() == "itemref"
+                    && let Some(idref) = node.attribute("idref")
+                    && let Some(href) = manifest.get(idref)
+                {
+                    spine_order.push(href.clone());
+                }
+            }
+
+            Ok((metadata, spine_order))
+        }
+        Err(e) => Err(crate::KreuzbergError::Parsing {
+            message: format!("Failed to parse OPF file: {}", e),
+            source: None,
+        }),
+    }
+}

data/vendor/kreuzberg/src/extractors/epub/mod.rs

@@ -0,0 +1,186 @@
+//! Native EPUB extractor using permissive-licensed dependencies.
+//!
+//! This extractor provides native Rust-based EPUB extraction without GPL-licensed
+//! dependencies, extracting:
+//! - Metadata from OPF (Open Packaging Format) using Dublin Core standards
+//! - Content from XHTML files in spine order
+//! - Proper handling of EPUB2 and EPUB3 formats
+//!
+//! Uses only permissive-licensed crates:
+//! - `zip` (MIT/Apache) - for reading EPUB container
+//! - `roxmltree` (MIT) - for parsing XML
+//! - `html-to-markdown-rs` (MIT) - for converting XHTML to plain text
+
+mod content;
+mod metadata;
+mod parsing;
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::{ExtractionResult, Metadata};
+use async_trait::async_trait;
+use std::io::Cursor;
+use zip::ZipArchive;
+
+use content::extract_content;
+use metadata::extract_metadata;
+use parsing::{parse_container_xml, read_file_from_zip};
+
+/// EPUB format extractor using permissive-licensed dependencies.
+///
+/// Extracts content and metadata from EPUB files (both EPUB2 and EPUB3)
+/// using native Rust parsing without GPL-licensed dependencies.
+pub struct EpubExtractor;
+
+impl EpubExtractor {
+    /// Create a new EPUB extractor.
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for EpubExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Plugin for EpubExtractor {
+    fn name(&self) -> &str {
+        "epub-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn description(&self) -> &str {
+        "Extracts content and metadata from EPUB documents (native Rust implementation with permissive licenses)"
+    }
+
+    fn author(&self) -> &str {
+        "Kreuzberg Team"
+    }
+}
+
+#[cfg(feature = "office")]
+#[async_trait]
+impl DocumentExtractor for EpubExtractor {
+    #[cfg_attr(
+        feature = "otel",
+        tracing::instrument(
+            skip(self, content, _config),
+            fields(
+                extractor.name = self.name(),
+                content.size_bytes = content.len(),
+            )
+        )
+    )]
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let cursor = Cursor::new(content.to_vec());
+
+        let mut archive = ZipArchive::new(cursor).map_err(|e| crate::KreuzbergError::Parsing {
+            message: format!("Failed to open EPUB as ZIP: {}", e),
+            source: None,
+        })?;
+
+        let container_xml = read_file_from_zip(&mut archive, "META-INF/container.xml")?;
+        let opf_path = parse_container_xml(&container_xml)?;
+
+        let manifest_dir = if let Some(last_slash) = opf_path.rfind('/') {
+            opf_path[..last_slash].to_string()
+        } else {
+            String::new()
+        };
+
+        let opf_xml = read_file_from_zip(&mut archive, &opf_path)?;
+
+        let extracted_content = extract_content(&mut archive, &opf_path, &manifest_dir)?;
+
+        let (epub_metadata, additional_metadata) = extract_metadata(&opf_xml)?;
+        let metadata_map: std::collections::HashMap<String, serde_json::Value> =
+            additional_metadata.into_iter().collect();
+
+        Ok(ExtractionResult {
+            content: extracted_content,
+            mime_type: mime_type.to_string(),
+            metadata: Metadata {
+                title: epub_metadata.title,
+                authors: epub_metadata.creator.map(|c| vec![c]),
+                language: epub_metadata.language,
+                created_at: epub_metadata.date,
+                additional: metadata_map,
+                ..Default::default()
+            },
+            pages: None,
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            djot_content: None,
+            elements: None,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &[
+            "application/epub+zip",
+            "application/x-epub+zip",
+            "application/vnd.epub+zip",
+        ]
+    }
+
+    fn priority(&self) -> i32 {
+        60
+    }
+}
+
+#[cfg(all(test, feature = "office"))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_epub_extractor_plugin_interface() {
+        let extractor = EpubExtractor::new();
+        assert_eq!(extractor.name(), "epub-extractor");
+        assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
+        assert_eq!(extractor.priority(), 60);
+        assert!(!extractor.supported_mime_types().is_empty());
+    }
+
+    #[test]
+    fn test_epub_extractor_default() {
+        let extractor = EpubExtractor;
+        assert_eq!(extractor.name(), "epub-extractor");
+    }
+
+    #[tokio::test]
+    async fn test_epub_extractor_initialize_shutdown() {
+        let extractor = EpubExtractor::new();
+        assert!(extractor.initialize().is_ok());
+        assert!(extractor.shutdown().is_ok());
+    }
+
+    #[test]
+    fn test_epub_extractor_supported_mime_types() {
+        let extractor = EpubExtractor::new();
+        let supported = extractor.supported_mime_types();
+        assert!(supported.contains(&"application/epub+zip"));
+        assert!(supported.contains(&"application/x-epub+zip"));
+        assert!(supported.contains(&"application/vnd.epub+zip"));
+    }
+}