kreuzberg 4.0.0.pre.rc.6

data/vendor/kreuzberg/src/extractors/fictionbook.rs.backup2

@@ -0,0 +1,738 @@
+//! FictionBook (FB2) document extractor supporting FictionBook 2.0 format.
+//!
+//! This extractor handles FictionBook XML documents (FB2), an XML-based e-book format
+//! popular in Russian-speaking countries.
+//!
+//! It extracts:
+//! - Document metadata (genre, language)
+//! - Section hierarchy and content
+//! - Paragraphs and text content with inline formatting
+//! - Inline markup: emphasis, strong, strikethrough, subscript, superscript, code
+//! - Blockquotes and notes
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::{ExtractionResult, Metadata};
+use async_trait::async_trait;
+use quick_xml::Reader;
+use quick_xml::events::Event;
+
+/// FictionBook document extractor.
+///
+/// Supports FictionBook 2.0 format with proper section hierarchy and inline formatting.
+pub struct FictionBookExtractor;
+
+impl Default for FictionBookExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl FictionBookExtractor {
+    pub fn new() -> Self {
+        Self
+    }
+
+    /// Extract paragraph content with markdown formatting preservation.
+    /// Handles inline formatting tags like emphasis (*), strong (**), strikethrough (~~), etc.
+    fn extract_paragraph_content(reader: &mut Reader<&[u8]>) -> Result<String> {
+        let mut text = String::new();
+        let mut para_depth = 0;
+
+        loop {
+            match reader.read_event() {
+                Ok(Event::Start(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    match tag.as_str() {
+                        "emphasis" => {
+                            text.push('*');
+                        }
+                        "strong" => {
+                            text.push_str("**");
+                        }
+                        "strikethrough" => {
+                            text.push_str("~~");
+                        }
+                        "code" => {
+                            text.push('`');
+                        }
+                        "sub" => {
+                            text.push('~');
+                        }
+                        "sup" => {
+                            text.push('^');
+                        }
+                        "a" | "empty-line" => {
+                            // Links and empty lines - just continue
+                        }
+                        _ => {}
+                    }
+                    para_depth += 1;
+                }
+                Ok(Event::End(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    match tag.as_str() {
+                        "p" if para_depth == 1 => {
+                            // End of paragraph
+                            break;
+                        }
+                        "emphasis" => {
+                            text.push('*');
+                        }
+                        "strong" => {
+                            text.push_str("**");
+                        }
+                        "strikethrough" => {
+                            text.push_str("~~");
+                        }
+                        "code" => {
+                            text.push('`');
+                        }
+                        "sub" => {
+                            text.push('~');
+                        }
+                        "sup" => {
+                            text.push('^');
+                        }
+                        _ => {}
+                    }
+                    if para_depth > 0 {
+                        para_depth -= 1;
+                    }
+                }
+                Ok(Event::Text(t)) => {
+                    let decoded = String::from_utf8_lossy(t.as_ref()).to_string();
+                    let trimmed = decoded.trim();
+                    if !trimmed.is_empty() {
+                        if !text.is_empty()
+                            && !text.ends_with(' ')
+                            && !text.ends_with('*')
+                            && !text.ends_with('`')
+                            && !text.ends_with('~')
+                            && !text.ends_with('^')
+                        {
+                            text.push(' ');
+                        }
+                        text.push_str(trimmed);
+                    }
+                }
+                Ok(Event::Eof) => break,
+                Err(e) => {
+                    return Err(crate::error::KreuzbergError::parsing(format!(
+                        "XML parsing error: {}",
+                        e
+                    )));
+                }
+                _ => {}
+            }
+        }
+
+        Ok(text.trim().to_string())
+    }
+
+    /// Extract epigraph content with proper formatting.
+    /// Epigraphs contain paragraph elements that should be extracted.
+    fn extract_epigraph_content(reader: &mut Reader<&[u8]>) -> Result<String> {
+        let mut content = String::new();
+        let mut epigraph_depth = 1;
+
+        loop {
+            match reader.read_event() {
+                Ok(Event::Start(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    match tag.as_str() {
+                        "p" => match Self::extract_paragraph_content(reader) {
+                            Ok(para) if !para.is_empty() => {
+                                content.push_str(&para);
+                                content.push('\n');
+                            }
+                            _ => {}
+                        },
+                        "epigraph" => {
+                            epigraph_depth += 1;
+                        }
+                        "empty-line" => {
+                            content.push('\n');
+                        }
+                        _ => {
+                            epigraph_depth += 1;
+                        }
+                    }
+                }
+                Ok(Event::End(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    match tag.as_str() {
+                        "epigraph" if epigraph_depth == 1 => {
+                            // End of epigraph
+                            break;
+                        }
+                        _ => {
+                            if epigraph_depth > 0 {
+                                epigraph_depth -= 1;
+                            }
+                        }
+                    }
+                }
+                Ok(Event::Eof) => break,
+                Err(e) => {
+                    return Err(crate::error::KreuzbergError::parsing(format!(
+                        "XML parsing error: {}",
+                        e
+                    )));
+                }
+                _ => {}
+            }
+        }
+
+        Ok(content.trim().to_string())
+    }
+
+    /// Extract text content from a FictionBook element and its children.
+    fn extract_text_content(reader: &mut Reader<&[u8]>) -> Result<String> {
+        let mut text = String::new();
+        let mut depth = 0;
+
+        loop {
+            match reader.read_event() {
+                Ok(Event::Start(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    // Handle formatting tags
+                    match tag.as_str() {
+                        "emphasis" | "strong" | "strikethrough" | "code" | "sub" | "sup" => {
+                            // These are inline formatting, just continue parsing
+                        }
+                        "empty-line" => {
+                            text.push('\n');
+                        }
+                        _ => {}
+                    }
+                    depth += 1;
+                }
+                Ok(Event::End(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    if depth == 0 {
+                        break;
+                    }
+                    depth -= 1;
+                    if (tag == "p" || tag == "cite" || tag == "section") && !text.is_empty() && !text.ends_with('\n') {
+                        text.push('\n');
+                    }
+                }
+                Ok(Event::Text(t)) => {
+                    let decoded = String::from_utf8_lossy(t.as_ref()).to_string();
+                    let trimmed = decoded.trim();
+                    if !trimmed.is_empty() {
+                        if !text.is_empty() && !text.ends_with(' ') && !text.ends_with('\n') {
+                            text.push(' ');
+                        }
+                        text.push_str(trimmed);
+                        text.push(' ');
+                    }
+                }
+                Ok(Event::CData(t)) => {
+                    let decoded = String::from_utf8_lossy(t.as_ref()).to_string();
+                    if !decoded.trim().is_empty() {
+                        if !text.is_empty() && !text.ends_with('\n') {
+                            text.push('\n');
+                        }
+                        text.push_str(&decoded);
+                        text.push('\n');
+                    }
+                }
+                Ok(Event::Eof) => break,
+                Err(e) => {
+                    return Err(crate::error::KreuzbergError::parsing(format!(
+                        "XML parsing error: {}",
+                        e
+                    )));
+                }
+                _ => {}
+            }
+        }
+
+        // Clean up extra whitespace and normalize
+        let text = text
+            .lines()
+            .map(|line| line.trim())
+            .filter(|line| !line.is_empty())
+            .collect::<Vec<_>>()
+            .join("\n");
+
+        Ok(text)
+    }
+
+    /// Extract metadata from FictionBook document.
+    fn extract_metadata(data: &[u8]) -> Result<Metadata> {
+        let mut reader = Reader::from_reader(data);
+        let mut metadata = Metadata::default();
+        let mut in_title_info = false;
+        let mut in_description = false;
+
+        loop {
+            match reader.read_event() {
+                Ok(Event::Start(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+
+                    match tag.as_str() {
+                        "description" => {
+                            in_description = true;
+                        }
+                        "title-info" if in_description => {
+                            in_title_info = true;
+                        }
+                        "genre" if in_title_info => {
+                            // Extract genre
+                            if let Ok(Event::Text(t)) = reader.read_event() {
+                                let genre = String::from_utf8_lossy(t.as_ref()).to_string();
+                                if !genre.trim().is_empty() && genre.trim() != "unrecognised" {
+                                    metadata.subject = Some(genre.trim().to_string());
+                                }
+                            }
+                        }
+                        "date" if in_title_info => {
+                            // Extract date
+                            if let Ok(Event::Text(t)) = reader.read_event() {
+                                let date = String::from_utf8_lossy(t.as_ref()).to_string();
+                                if !date.trim().is_empty() {
+                                    metadata.date = Some(date.trim().to_string());
+                                }
+                            }
+                        }
+                        "lang" if in_title_info => {
+                            // Extract language
+                            if let Ok(Event::Text(t)) = reader.read_event() {
+                                let lang = String::from_utf8_lossy(t.as_ref()).to_string();
+                                if !lang.trim().is_empty() {
+                                    metadata.language = Some(lang.trim().to_string());
+                                }
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+                Ok(Event::End(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    if tag == "title-info" {
+                        in_title_info = false;
+                    } else if tag == "description" {
+                        in_description = false;
+                    }
+                }
+                Ok(Event::Eof) => break,
+                Err(_) => break,
+                _ => {}
+            }
+        }
+
+        Ok(metadata)
+    }
+
+    /// Extract content from FictionBook document body sections.
+    fn extract_body_content(data: &[u8]) -> Result<String> {
+        let mut reader = Reader::from_reader(data);
+        let mut content = String::new();
+        let mut in_body = false;
+        let mut skip_notes_body = false;
+
+        loop {
+            match reader.read_event() {
+                Ok(Event::Start(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+
+                    if tag == "body" {
+                        // Check if it's the notes body
+                        for a in e.attributes().flatten() {
+                            let attr_name = String::from_utf8_lossy(a.key.as_ref()).to_string();
+                            if attr_name == "name" {
+                                let val = String::from_utf8_lossy(a.value.as_ref());
+                                if val == "notes" {
+                                    skip_notes_body = true;
+                                    break;
+                                }
+                            }
+                        }
+
+                        if !skip_notes_body {
+                            in_body = true;
+                        }
+                    } else if tag == "section" && in_body {
+                        // Extract section content
+                        match Self::extract_section_content(&mut reader) {
+                            Ok(section_content) if !section_content.is_empty() => {
+                                content.push_str(&section_content);
+                                content.push('\n');
+                            }
+                            _ => {}
+                        }
+                    } else if tag == "p" && in_body && !skip_notes_body {
+                        // Extract paragraph content with formatting preservation
+                        match Self::extract_paragraph_content(&mut reader) {
+                            Ok(para) if !para.is_empty() => {
+                                content.push_str(&para);
+                                content.push('\n');
+                            }
+                            _ => {}
+                        }
+                    } else if tag == "epigraph" && in_body && !skip_notes_body {
+                        // Extract epigraph content with blank line after
+                        match Self::extract_epigraph_content(&mut reader) {
+                            Ok(epigraph_content) if !epigraph_content.is_empty() => {
+                                content.push_str(&epigraph_content);
+                                content.push_str("\n\n");
+                            }
+                            _ => {}
+                        }
+                    } else if tag == "title" && in_body {
+                        // Extract title from body
+                        match Self::extract_text_content(&mut reader) {
+                            Ok(title_content) if !title_content.is_empty() => {
+                                content.push_str(&format!("# {}\n", title_content));
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+                Ok(Event::End(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    if tag == "body" {
+                        if skip_notes_body {
+                            skip_notes_body = false;
+                        } else {
+                            in_body = false;
+                        }
+                    }
+                }
+                Ok(Event::Eof) => break,
+                Err(_) => break,
+                _ => {}
+            }
+        }
+
+        Ok(content.trim().to_string())
+    }
+
+    /// Extract verse content from a poem stanza.
+    fn extract_verse_content(reader: &mut Reader<&[u8]>) -> Result<String> {
+        let mut text = String::new();
+        let mut verse_depth = 0;
+
+        loop {
+            match reader.read_event() {
+                Ok(Event::Start(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    match tag.as_str() {
+                        "emphasis" | "strong" | "strikethrough" | "code" | "sub" | "sup" => {
+                            // Pass through formatting
+                            verse_depth += 1;
+                        }
+                        _ => {
+                            verse_depth += 1;
+                        }
+                    }
+                }
+                Ok(Event::End(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    if tag == "v" && verse_depth == 1 {
+                        break;
+                    }
+                    if verse_depth > 0 {
+                        verse_depth -= 1;
+                    }
+                }
+                Ok(Event::Text(t)) => {
+                    let decoded = String::from_utf8_lossy(t.as_ref()).to_string();
+                    let trimmed = decoded.trim();
+                    if !trimmed.is_empty() {
+                        if !text.is_empty() && !text.ends_with(' ') {
+                            text.push(' ');
+                        }
+                        text.push_str(trimmed);
+                    }
+                }
+                Ok(Event::Eof) => break,
+                Err(_) => break,
+                _ => {}
+            }
+        }
+
+        Ok(text.trim().to_string())
+    }
+
+    /// Extract stanza content from a poem (handles multiple verses).
+    fn extract_stanza_content(reader: &mut Reader<&[u8]>) -> Result<String> {
+        let mut content = String::new();
+        let mut stanza_depth = 1;
+
+        loop {
+            match reader.read_event() {
+                Ok(Event::Start(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+
+                    match tag.as_str() {
+                        "v" => {
+                            match Self::extract_verse_content(reader) {
+                                Ok(verse) if !verse.is_empty() => {
+                                    content.push_str(&verse);
+                                    content.push('\n');
+                                }
+                                _ => {}
+                            }
+                        }
+                        "title" | "subtitle" => {
+                            match Self::extract_text_content(reader) {
+                                Ok(text) if !text.is_empty() => {
+                                    content.push_str(&text);
+                                    content.push('\n');
+                                }
+                                _ => {}
+                            }
+                        }
+                        _ => {
+                            stanza_depth += 1;
+                        }
+                    }
+                }
+                Ok(Event::End(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    if tag == "stanza" {
+                        stanza_depth -= 1;
+                        if stanza_depth == 0 {
+                            break;
+                        }
+                    }
+                }
+                Ok(Event::Eof) => break,
+                Err(_) => break,
+                _ => {}
+            }
+        }
+
+        Ok(content.trim().to_string())
+    }
+
+    /// Extract poem content (handles title, stanzas, author, date).
+    fn extract_poem_content(reader: &mut Reader<&[u8]>) -> Result<String> {
+        let mut content = String::new();
+        let mut poem_depth = 1;
+
+        loop {
+            match reader.read_event() {
+                Ok(Event::Start(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+
+                    match tag.as_str() {
+                        "title" | "epigraph" | "text-author" | "date" => {
+                            match Self::extract_text_content(reader) {
+                                Ok(text) if !text.is_empty() => {
+                                    content.push_str(&text);
+                                    content.push('\n');
+                                }
+                                _ => {}
+                            }
+                        }
+                        "stanza" => {
+                            match Self::extract_stanza_content(reader) {
+                                Ok(stanza) if !stanza.is_empty() => {
+                                    content.push_str(&stanza);
+                                    content.push('\n');
+                                }
+                                _ => {}
+                            }
+                        }
+                        _ => {
+                            poem_depth += 1;
+                        }
+                    }
+                }
+                Ok(Event::End(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    if tag == "poem" {
+                        poem_depth -= 1;
+                        if poem_depth == 0 {
+                            break;
+                        }
+                    }
+                }
+                Ok(Event::Eof) => break,
+                Err(_) => break,
+                _ => {}
+            }
+        }
+
+        Ok(content.trim().to_string())
+    }
+
+    /// Extract content from a section with proper hierarchy.
+    fn extract_section_content(reader: &mut Reader<&[u8]>) -> Result<String> {
+        let mut content = String::new();
+        let mut section_depth = 1;
+
+        loop {
+            match reader.read_event() {
+                Ok(Event::Start(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+
+                    match tag.as_str() {
+                        "section" => {
+                            section_depth += 1;
+                        }
+                        "title" => match Self::extract_text_content(reader) {
+                            Ok(title_text) if !title_text.is_empty() => {
+                                let heading_level = std::cmp::min(section_depth + 1, 6);
+                                let heading = "#".repeat(heading_level);
+                                content.push_str(&format!("{} {}\n", heading, title_text));
+                            }
+                            _ => {}
+                        },
+                        "p" => match Self::extract_paragraph_content(reader) {
+                            Ok(para) if !para.is_empty() => {
+                                content.push_str(&para);
+                                content.push('\n');
+                            }
+                            _ => {}
+                        },
+                        "cite" => match Self::extract_text_content(reader) {
+                            Ok(cite_content) if !cite_content.is_empty() => {
+                                content.push_str("> ");
+                                content.push_str(&cite_content);
+                                content.push('\n');
+                            }
+                            _ => {}
+                        },
+                        "poem" => match Self::extract_poem_content(reader) {
+                            Ok(poem_content) if !poem_content.is_empty() => {
+                                content.push_str(&poem_content);
+                                content.push('\n');
+                            }
+                            _ => {}
+                        },
+                        "epigraph" => match Self::extract_epigraph_content(reader) {
+                            Ok(epigraph_content) if !epigraph_content.is_empty() => {
+                                content.push_str(&epigraph_content);
+                                content.push_str("\n\n");
+                            }
+                            _ => {}
+                        },
+                        "empty-line" => {
+                            content.push('\n');
+                        }
+                        _ => {}
+                    }
+                }
+                Ok(Event::End(e)) => {
+                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
+                    if tag == "section" {
+                        section_depth -= 1;
+                        if section_depth == 0 {
+                            break;
+                        }
+                    }
+                }
+                Ok(Event::Eof) => break,
+                Err(_) => break,
+                _ => {}
+            }
+        }
+
+        Ok(content.trim().to_string())
+    }
+}
+
+impl Plugin for FictionBookExtractor {
+    fn name(&self) -> &str {
+        "fictionbook-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn description(&self) -> &str {
+        "Extracts content and metadata from FictionBook documents (FB2 format)"
+    }
+
+    fn author(&self) -> &str {
+        "Kreuzberg Team"
+    }
+}
+
+#[async_trait]
+impl DocumentExtractor for FictionBookExtractor {
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        // Extract metadata
+        let metadata = Self::extract_metadata(content)?;
+
+        // Extract body content
+        let extracted_content = Self::extract_body_content(content)?;
+
+        Ok(ExtractionResult {
+            content: extracted_content,
+            mime_type: mime_type.to_string(),
+            metadata,
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &[
+            "application/x-fictionbook+xml",
+            "text/x-fictionbook",
+            "application/x-fictionbook",
+        ]
+    }
+
+    fn priority(&self) -> i32 {
+        50
+    }
+}
+
+#[cfg(all(test, feature = "office"))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_fictionbook_extractor_plugin_interface() {
+        let extractor = FictionBookExtractor::new();
+        assert_eq!(extractor.name(), "fictionbook-extractor");
+        assert_eq!(extractor.priority(), 50);
+        assert!(!extractor.supported_mime_types().is_empty());
+    }
+
+    #[test]
+    fn test_fictionbook_extractor_default() {
+        let extractor = FictionBookExtractor::default();
+        assert_eq!(extractor.name(), "fictionbook-extractor");
+    }
+
+    #[test]
+    fn test_fictionbook_extractor_supported_mime_types() {
+        let extractor = FictionBookExtractor::new();
+        let supported = extractor.supported_mime_types();
+        assert!(supported.contains(&"application/x-fictionbook+xml"));
+        assert!(supported.contains(&"text/x-fictionbook"));
+    }
+
+    #[tokio::test]
+    async fn test_fictionbook_extractor_initialize_shutdown() {
+        let extractor = FictionBookExtractor::new();
+        assert!(extractor.initialize().is_ok());
+        assert!(extractor.shutdown().is_ok());
+    }
+}