kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/extractors/{jats.rs → jats/mod.rs}

@@ -13,17 +13,20 @@
 //! - Citations and references
 //! - Supplementary material information
 
+mod elements;
+mod metadata;
+mod parser;
+
 use crate::Result;
 use crate::core::config::ExtractionConfig;
-use crate::extraction::cells_to_markdown;
 use crate::plugins::{DocumentExtractor, Plugin};
-use crate::types::{ExtractionResult, Metadata, Table};
+use crate::types::{ExtractionResult, Metadata};
 use async_trait::async_trait;
-use quick_xml::Reader;
-use quick_xml::events::Event;
 #[cfg(feature = "tokio-runtime")]
 use std::path::Path;
 
+use elements::extract_jats_all_in_one;
+
 /// JATS document extractor.
 ///
 /// Supports JATS (Journal Article Tag Suite) XML documents in various versions,
@@ -42,414 +45,6 @@ impl JatsExtractor {
     }
 }
 
-/// Extract text content from a JATS element and its children.
-fn extract_text_content(reader: &mut Reader<&[u8]>) -> Result<String> {
-    let mut text = String::new();
-    let mut depth = 0;
-
-    loop {
-        match reader.read_event() {
-            Ok(Event::Start(_)) => {
-                depth += 1;
-            }
-            Ok(Event::End(_)) => {
-                if depth == 0 {
-                    break;
-                }
-                depth -= 1;
-                if !text.is_empty() && !text.ends_with('\n') {
-                    text.push(' ');
-                }
-            }
-            Ok(Event::Text(t)) => {
-                let decoded = String::from_utf8_lossy(t.as_ref()).to_string();
-                if !decoded.trim().is_empty() {
-                    text.push_str(&decoded);
-                    text.push(' ');
-                }
-            }
-            Ok(Event::CData(t)) => {
-                let decoded = std::str::from_utf8(t.as_ref()).unwrap_or("").to_string();
-                if !decoded.trim().is_empty() {
-                    text.push_str(&decoded);
-                    text.push('\n');
-                }
-            }
-            Ok(Event::Eof) => break,
-            Err(e) => {
-                return Err(crate::error::KreuzbergError::parsing(format!(
-                    "XML parsing error: {}",
-                    e
-                )));
-            }
-            _ => {}
-        }
-    }
-
-    Ok(text.trim().to_string())
-}
-
-/// Structure to hold extracted JATS metadata.
-#[derive(Debug, Clone, Default)]
-struct JatsMetadataExtracted {
-    title: String,
-    subtitle: Option<String>,
-    authors: Vec<String>,
-    affiliations: Vec<String>,
-    doi: Option<String>,
-    pii: Option<String>,
-    keywords: Vec<String>,
-    publication_date: Option<String>,
-    volume: Option<String>,
-    issue: Option<String>,
-    pages: Option<String>,
-    journal_title: Option<String>,
-    article_type: Option<String>,
-    abstract_text: Option<String>,
-    corresponding_author: Option<String>,
-}
-
-/// Extract all content in a single optimized pass.
-/// Combines metadata extraction, content parsing, and table extraction into one pass.
-fn extract_jats_all_in_one(content: &str) -> Result<(JatsMetadataExtracted, String, String, Vec<Table>)> {
-    let mut reader = Reader::from_str(content);
-    let mut metadata = JatsMetadataExtracted::default();
-    let mut body_content = String::new();
-    let mut title = String::new();
-
-    let mut in_article_meta = false;
-    let mut in_article_title = false;
-    let mut in_subtitle = false;
-    let mut in_contrib = false;
-    let mut in_name = false;
-    let mut in_aff = false;
-    let mut in_abstract = false;
-    let mut in_kwd_group = false;
-    let mut in_kwd = false;
-    let mut current_author = String::new();
-    let mut current_aff = String::new();
-    let mut abstract_content = String::new();
-
-    let mut in_body = false;
-    let mut in_section = false;
-    let mut in_para = false;
-
-    let mut in_table = false;
-    let mut in_thead = false;
-    let mut in_tbody = false;
-    let mut in_row = false;
-    let mut current_table: Vec<Vec<String>> = Vec::new();
-    let mut current_row: Vec<String> = Vec::new();
-    let mut tables = Vec::new();
-    let mut table_index = 0;
-
-    loop {
-        match reader.read_event() {
-            Ok(Event::Start(e)) => {
-                let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
-
-                match tag.as_str() {
-                    "article" => {
-                        for attr in e.attributes() {
-                            if let Ok(attr) = attr
-                                && String::from_utf8_lossy(attr.key.as_ref()) == "article-type"
-                            {
-                                metadata.article_type = Some(String::from_utf8_lossy(attr.value.as_ref()).to_string());
-                            }
-                        }
-                    }
-                    "article-meta" => {
-                        in_article_meta = true;
-                    }
-                    "article-title" if in_article_meta => {
-                        in_article_title = true;
-                    }
-                    "subtitle" if in_article_meta => {
-                        in_subtitle = true;
-                    }
-                    "contrib" if in_article_meta => {
-                        in_contrib = true;
-                        current_author.clear();
-                    }
-                    "name" if in_contrib => {
-                        in_name = true;
-                    }
-                    "aff" if in_article_meta => {
-                        in_aff = true;
-                        current_aff.clear();
-                    }
-                    "article-id" if in_article_meta => {
-                        let mut id_type = String::new();
-                        for attr in e.attributes() {
-                            if let Ok(attr) = attr
-                                && String::from_utf8_lossy(attr.key.as_ref()) == "pub-id-type"
-                            {
-                                id_type = String::from_utf8_lossy(attr.value.as_ref()).to_string();
-                            }
-                        }
-
-                        let id_text = extract_text_content(&mut reader)?;
-                        match id_type.as_str() {
-                            "doi" => metadata.doi = Some(id_text),
-                            "pii" => metadata.pii = Some(id_text),
-                            _ => {}
-                        }
-                        continue;
-                    }
-                    "volume" if in_article_meta => {
-                        let vol_text = extract_text_content(&mut reader)?;
-                        metadata.volume = Some(vol_text);
-                        continue;
-                    }
-                    "issue" if in_article_meta => {
-                        let issue_text = extract_text_content(&mut reader)?;
-                        metadata.issue = Some(issue_text);
-                        continue;
-                    }
-                    "fpage" | "lpage" if in_article_meta => {
-                        let page_text = extract_text_content(&mut reader)?;
-                        if let Some(pages) = &mut metadata.pages {
-                            pages.push('-');
-                            pages.push_str(&page_text);
-                        } else {
-                            metadata.pages = Some(page_text);
-                        }
-                        continue;
-                    }
-                    "pub-date" if in_article_meta => {
-                        let date_text = extract_text_content(&mut reader)?;
-                        if metadata.publication_date.is_none() {
-                            metadata.publication_date = Some(date_text);
-                        }
-                        continue;
-                    }
-                    "journal-title" if in_article_meta => {
-                        let journal_text = extract_text_content(&mut reader)?;
-                        if metadata.journal_title.is_none() {
-                            metadata.journal_title = Some(journal_text);
-                        }
-                        continue;
-                    }
-                    "abstract" if in_article_meta => {
-                        in_abstract = true;
-                        abstract_content.clear();
-                    }
-                    "kwd-group" if in_article_meta => {
-                        in_kwd_group = true;
-                    }
-                    "kwd" if in_kwd_group => {
-                        in_kwd = true;
-                    }
-                    "corresp" if in_article_meta => {
-                        let corresp_text = extract_text_content(&mut reader)?;
-                        metadata.corresponding_author = Some(corresp_text);
-                        continue;
-                    }
-                    "body" => {
-                        in_body = true;
-                    }
-                    "sec" if in_body => {
-                        in_section = true;
-                    }
-                    "title" if (in_section || in_body) && !in_article_title => {
-                        let section_title = extract_text_content(&mut reader)?;
-                        if !section_title.is_empty() {
-                            body_content.push_str("## ");
-                            body_content.push_str(&section_title);
-                            body_content.push_str("\n\n");
-                        }
-                        continue;
-                    }
-                    "p" if in_body || in_section => {
-                        in_para = true;
-                    }
-                    "table" => {
-                        in_table = true;
-                        current_table.clear();
-                    }
-                    "thead" if in_table => {
-                        in_thead = true;
-                    }
-                    "tbody" if in_table => {
-                        in_tbody = true;
-                    }
-                    "tr" if (in_thead || in_tbody) && in_table => {
-                        in_row = true;
-                        current_row.clear();
-                    }
-                    "td" | "th" if in_row => {
-                        let mut cell_text = String::new();
-                        let mut cell_depth = 0;
-
-                        loop {
-                            match reader.read_event() {
-                                Ok(Event::Start(_)) => {
-                                    cell_depth += 1;
-                                }
-                                Ok(Event::End(e)) => {
-                                    let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
-                                    if (tag == "td" || tag == "th") && cell_depth == 0 {
-                                        break;
-                                    }
-                                    if cell_depth > 0 {
-                                        cell_depth -= 1;
-                                    }
-                                }
-                                Ok(Event::Text(t)) => {
-                                    let decoded = String::from_utf8_lossy(t.as_ref()).to_string();
-                                    if !decoded.trim().is_empty() {
-                                        if !cell_text.is_empty() {
-                                            cell_text.push(' ');
-                                        }
-                                        cell_text.push_str(decoded.trim());
-                                    }
-                                }
-                                Ok(Event::Eof) => break,
-                                Err(e) => {
-                                    return Err(crate::error::KreuzbergError::parsing(format!(
-                                        "XML parsing error: {}",
-                                        e
-                                    )));
-                                }
-                                _ => {}
-                            }
-                        }
-
-                        current_row.push(cell_text);
-                    }
-                    _ => {}
-                }
-            }
-            Ok(Event::End(e)) => {
-                let tag = String::from_utf8_lossy(e.name().as_ref()).to_string();
-
-                match tag.as_str() {
-                    "article-meta" => {
-                        in_article_meta = false;
-                    }
-                    "article-title" if in_article_title => {
-                        in_article_title = false;
-                    }
-                    "subtitle" if in_subtitle => {
-                        in_subtitle = false;
-                    }
-                    "contrib" if in_contrib => {
-                        if !current_author.is_empty() {
-                            metadata.authors.push(current_author.clone());
-                        }
-                        in_contrib = false;
-                        current_author.clear();
-                    }
-                    "name" if in_name => {
-                        in_name = false;
-                    }
-                    "aff" if in_aff => {
-                        if !current_aff.is_empty() {
-                            metadata.affiliations.push(current_aff.clone());
-                        }
-                        in_aff = false;
-                        current_aff.clear();
-                    }
-                    "abstract" if in_abstract => {
-                        in_abstract = false;
-                        metadata.abstract_text = Some(abstract_content.trim().to_string());
-                    }
-                    "kwd-group" if in_kwd_group => {
-                        in_kwd_group = false;
-                    }
-                    "kwd" if in_kwd => {
-                        in_kwd = false;
-                    }
-                    "body" => {
-                        in_body = false;
-                    }
-                    "sec" if in_section => {
-                        in_section = false;
-                    }
-                    "p" if in_para => {
-                        in_para = false;
-                    }
-                    "table" if in_table => {
-                        if !current_table.is_empty() {
-                            let markdown = cells_to_markdown(&current_table);
-                            tables.push(Table {
-                                cells: current_table.clone(),
-                                markdown,
-                                page_number: table_index + 1,
-                            });
-                            table_index += 1;
-                            current_table.clear();
-                        }
-                        in_table = false;
-                    }
-                    "thead" if in_thead => {
-                        in_thead = false;
-                    }
-                    "tbody" if in_tbody => {
-                        in_tbody = false;
-                    }
-                    "tr" if in_row => {
-                        if !current_row.is_empty() {
-                            current_table.push(current_row.clone());
-                            current_row.clear();
-                        }
-                        in_row = false;
-                    }
-                    _ => {}
-                }
-            }
-            Ok(Event::Text(t)) => {
-                let decoded = String::from_utf8_lossy(t.as_ref()).to_string();
-                let trimmed = decoded.trim();
-
-                if !trimmed.is_empty() {
-                    if in_article_title && metadata.title.is_empty() {
-                        metadata.title.push_str(trimmed);
-                    } else if in_subtitle && metadata.subtitle.is_none() {
-                        metadata.subtitle = Some(trimmed.to_string());
-                    } else if in_name {
-                        if !current_author.is_empty() {
-                            current_author.push(' ');
-                        }
-                        current_author.push_str(trimmed);
-                    } else if in_aff {
-                        if !current_aff.is_empty() {
-                            current_aff.push(' ');
-                        }
-                        current_aff.push_str(trimmed);
-                    } else if in_abstract {
-                        if !abstract_content.is_empty() {
-                            abstract_content.push(' ');
-                        }
-                        abstract_content.push_str(trimmed);
-                    } else if in_kwd {
-                        metadata.keywords.push(trimmed.to_string());
-                    } else if in_para && in_body {
-                        body_content.push_str(trimmed);
-                        body_content.push_str("\n\n");
-                    }
-                }
-            }
-            Ok(Event::Eof) => break,
-            Err(e) => {
-                return Err(crate::error::KreuzbergError::parsing(format!(
-                    "XML parsing error: {}",
-                    e
-                )));
-            }
-            _ => {}
-        }
-    }
-
-    let mut final_output = body_content;
-    if !metadata.title.is_empty() {
-        final_output = format!("# {}\n\n{}", metadata.title, final_output);
-        title = metadata.title.clone();
-    }
-
-    Ok((metadata, final_output.trim().to_string(), title, tables))
-}
-
 impl Plugin for JatsExtractor {
     fn name(&self) -> &str {
         "jats-extractor"
@@ -571,6 +166,8 @@ impl DocumentExtractor for JatsExtractor {
             chunks: None,
             images: None,
             pages: None,
+            djot_content: None,
+            elements: None,
         })
     }
 
@@ -602,6 +199,7 @@ impl DocumentExtractor for JatsExtractor {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use elements::extract_jats_all_in_one;
 
     #[test]
     fn test_jats_extractor_plugin_interface() {

data/vendor/kreuzberg/src/extractors/jats/parser.rs

@@ -0,0 +1,52 @@
+//! XML parsing and document structure traversal for JATS documents.
+
+use crate::Result;
+use quick_xml::Reader;
+use quick_xml::events::Event;
+
+/// Extract text content from a JATS element and its children.
+pub(super) fn extract_text_content(reader: &mut Reader<&[u8]>) -> Result<String> {
+    let mut text = String::new();
+    let mut depth = 0;
+
+    loop {
+        match reader.read_event() {
+            Ok(Event::Start(_)) => {
+                depth += 1;
+            }
+            Ok(Event::End(_)) => {
+                if depth == 0 {
+                    break;
+                }
+                depth -= 1;
+                if !text.is_empty() && !text.ends_with('\n') {
+                    text.push(' ');
+                }
+            }
+            Ok(Event::Text(t)) => {
+                let decoded = String::from_utf8_lossy(t.as_ref()).to_string();
+                if !decoded.trim().is_empty() {
+                    text.push_str(&decoded);
+                    text.push(' ');
+                }
+            }
+            Ok(Event::CData(t)) => {
+                let decoded = std::str::from_utf8(t.as_ref()).unwrap_or("").to_string();
+                if !decoded.trim().is_empty() {
+                    text.push_str(&decoded);
+                    text.push('\n');
+                }
+            }
+            Ok(Event::Eof) => break,
+            Err(e) => {
+                return Err(crate::error::KreuzbergError::parsing(format!(
+                    "XML parsing error: {}",
+                    e
+                )));
+            }
+            _ => {}
+        }
+    }
+
+    Ok(text.trim().to_string())
+}

data/vendor/kreuzberg/src/extractors/jupyter.rs

@@ -341,6 +341,8 @@ impl DocumentExtractor for JupyterExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
+            elements: None,
         })
     }
 

data/vendor/kreuzberg/src/extractors/latex/commands.rs

@@ -0,0 +1,93 @@
+//! LaTeX command processing.
+//!
+//! This module handles inline LaTeX commands like formatting (\textbf, \emph, etc.),
+//! math mode ($...$), and other inline elements.
+
+use super::utilities::read_braced_from_chars;
+
+/// Processes a line of LaTeX, handling commands and inline math.
+///
+/// Recursively processes nested commands and preserves math mode content.
+pub fn process_line(line: &str) -> String {
+    let mut result = String::new();
+    let mut chars = line.chars().peekable();
+
+    while let Some(ch) = chars.next() {
+        if ch == '\\' {
+            let mut cmd = String::new();
+            while let Some(&c) = chars.peek() {
+                if c.is_alphabetic() {
+                    cmd.push(chars.next().unwrap());
+                } else {
+                    break;
+                }
+            }
+
+            process_command(&cmd, &mut chars, &mut result);
+        } else if ch == '$' {
+            // Handle inline math
+            result.push(ch);
+            while let Some(&c) = chars.peek() {
+                result.push(chars.next().unwrap());
+                if c == '$' {
+                    break;
+                }
+            }
+        } else {
+            result.push(ch);
+        }
+    }
+
+    result
+}
+
+/// Processes a single LaTeX command.
+///
+/// Handles formatting commands (\textbf, \emph, etc.) and extracts their content.
+fn process_command(cmd: &str, chars: &mut std::iter::Peekable<std::str::Chars>, result: &mut String) {
+    match cmd {
+        "textbf" => {
+            if let Some(content) = read_braced_from_chars(chars) {
+                let processed = process_line(&content);
+                result.push_str(&processed);
+            }
+        }
+        "textit" | "emph" => {
+            if let Some(content) = read_braced_from_chars(chars) {
+                let processed = process_line(&content);
+                result.push_str(&processed);
+            }
+        }
+        "texttt" => {
+            if let Some(content) = read_braced_from_chars(chars) {
+                result.push_str(&content);
+            }
+        }
+        "underline" => {
+            if let Some(content) = read_braced_from_chars(chars) {
+                let processed = process_line(&content);
+                result.push_str(&processed);
+            }
+        }
+        "font" => {
+            // Skip font commands
+            while let Some(&c) = chars.peek() {
+                if c == '\\' {
+                    break;
+                }
+                chars.next();
+            }
+        }
+        "usepackage" => {
+            // Skip package declarations
+            read_braced_from_chars(chars);
+        }
+        _ => {
+            // For unknown commands, try to extract and process content
+            if let Some(content) = read_braced_from_chars(chars) {
+                let processed = process_line(&content);
+                result.push_str(&processed);
+            }
+        }
+    }
+}