kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/extractors/latex/environments.rs

@@ -0,0 +1,157 @@
+//! LaTeX environment processing.
+//!
+//! This module handles LaTeX environments like itemize, enumerate, description,
+//! tabular, and table environments.
+
+use super::commands::process_line;
+use super::utilities::{clean_text, collect_environment, extract_braced, extract_env_name};
+use crate::types::Table;
+
+/// Processes a list environment (itemize, enumerate, or description).
+///
+/// Converts LaTeX lists into markdown-style lists with proper nesting.
+pub fn process_list(content: &str, list_type: &str, output: &mut String) {
+    let lines: Vec<&str> = content.lines().collect();
+    let mut item_num = 1;
+    let mut i = 0;
+
+    while i < lines.len() {
+        let line = lines[i];
+        let trimmed = line.trim();
+
+        // Handle nested lists
+        if trimmed.contains("\\begin{")
+            && let Some(env_name) = extract_env_name(trimmed)
+            && (env_name == "itemize" || env_name == "enumerate" || env_name == "description")
+        {
+            let (nested_content, new_i) = collect_environment(&lines, i, &env_name);
+            let current_output_len = output.len();
+            process_list(&nested_content, &env_name, output);
+            let nested_output = output[current_output_len..].to_string();
+            output.truncate(current_output_len);
+            // Indent nested list
+            for nested_line in nested_output.lines() {
+                output.push_str("  ");
+                output.push_str(nested_line);
+                output.push('\n');
+            }
+            i = new_i;
+            continue;
+        }
+
+        // Handle \item
+        if trimmed.starts_with("\\item")
+            && let Some(pos) = trimmed.find("\\item")
+        {
+            let after = trimmed[pos + 5..].trim();
+
+            // Handle \item[label] for description lists
+            if after.starts_with('[')
+                && let Some(bracket_end) = after.find(']')
+            {
+                let label = after[1..bracket_end].to_string();
+                let text = after[bracket_end + 1..].trim().to_string();
+                if list_type == "description" {
+                    let processed_text = process_line(&text);
+                    output.push_str(&format!("{}: {}\n", label, processed_text));
+                    item_num += 1;
+                    i += 1;
+                    continue;
+                }
+            }
+
+            // Regular list item
+            let prefix = if list_type == "enumerate" {
+                format!("{}. ", item_num)
+            } else {
+                "- ".to_string()
+            };
+            output.push_str(&prefix);
+
+            let item_text = process_line(after);
+            output.push_str(item_text.trim());
+            output.push('\n');
+            item_num += 1;
+        }
+
+        i += 1;
+    }
+    output.push('\n');
+}
+
+/// Processes a tabular environment.
+///
+/// Converts LaTeX tables into markdown tables and creates Table structures.
+pub fn process_table(content: &str, output: &mut String, tables: &mut Vec<Table>) {
+    let lines: Vec<&str> = content.lines().collect();
+    let mut rows: Vec<Vec<String>> = Vec::new();
+
+    for line in lines {
+        let trimmed = line.trim();
+        if trimmed.starts_with("\\hline") || trimmed.is_empty() || trimmed.contains("\\begin{tabular}") {
+            continue;
+        }
+
+        let row_str = trimmed.replace("\\\\", "");
+        let cells: Vec<String> = row_str
+            .split('&')
+            .map(|s| clean_text(s.trim()))
+            .filter(|s| !s.is_empty())
+            .collect();
+
+        if !cells.is_empty() {
+            rows.push(cells);
+        }
+    }
+
+    if !rows.is_empty() {
+        let mut markdown = String::new();
+        for (i, row) in rows.iter().enumerate() {
+            markdown.push('|');
+            for cell in row {
+                markdown.push_str(&format!(" {} |", cell));
+            }
+            markdown.push('\n');
+
+            // Add header separator after first row
+            if i == 0 && rows.len() > 1 {
+                markdown.push('|');
+                for _ in row {
+                    markdown.push_str(" --- |");
+                }
+                markdown.push('\n');
+            }
+        }
+
+        output.push_str(&markdown);
+
+        let table = Table {
+            cells: rows,
+            markdown: markdown.clone(),
+            page_number: 1,
+        };
+        tables.push(table);
+    }
+}
+
+/// Processes a table environment with caption.
+///
+/// Extracts the caption and processes the embedded tabular environment.
+pub fn process_table_with_caption(content: &str, output: &mut String, tables: &mut Vec<Table>) {
+    // Extract and add caption if present
+    if content.contains("\\caption{")
+        && let Some(caption) = extract_braced(content, "caption")
+    {
+        output.push_str(&caption);
+        output.push('\n');
+    }
+
+    // Process the tabular environment inside
+    if content.contains("\\begin{tabular}")
+        && let Some(start) = content.find("\\begin{tabular}")
+        && let Some(end) = content.find("\\end{tabular}")
+    {
+        let tabular_content = &content[start..end + 13];
+        process_table(tabular_content, output, tables);
+    }
+}

data/vendor/kreuzberg/src/extractors/latex/metadata.rs

@@ -0,0 +1,27 @@
+//! Metadata extraction for LaTeX documents.
+//!
+//! This module handles extraction of document metadata like title, author, and date
+//! from LaTeX preamble commands.
+
+use super::utilities::extract_braced;
+use crate::types::Metadata;
+
+/// Extracts metadata from a LaTeX line.
+///
+/// Looks for \title{}, \author{}, and \date{} commands and populates
+/// the provided Metadata structure.
+pub fn extract_metadata_from_line(line: &str, metadata: &mut Metadata) {
+    if line.starts_with("\\title{") {
+        if let Some(title) = extract_braced(line, "title") {
+            metadata.additional.insert("title".to_string(), title.into());
+        }
+    } else if line.starts_with("\\author{") {
+        if let Some(author) = extract_braced(line, "author") {
+            metadata.additional.insert("author".to_string(), author.into());
+        }
+    } else if line.starts_with("\\date{")
+        && let Some(date) = extract_braced(line, "date")
+    {
+        metadata.additional.insert("date".to_string(), date.into());
+    }
+}

data/vendor/kreuzberg/src/extractors/latex/mod.rs

@@ -0,0 +1,146 @@
+//! Native Rust LaTeX text extractor.
+//!
+//! This extractor provides comprehensive LaTeX document parsing and text extraction.
+//!
+//! Features:
+//! - Metadata extraction: title, author, date from \title{}, \author{}, \date{}
+//! - Section hierarchy: \section{}, \subsection{}, \subsubsection{}, etc.
+//! - Inline formatting: \emph{}, \textbf{}, \textit{}, \texttt{}, \underline{}
+//! - Lists: itemize, enumerate, description environments
+//! - Tables: tabular environment parsing
+//! - Math: inline ($...$) and display (\[...\]) math preservation
+//! - Unicode support
+//!
+//! Requires the `office` feature.
+
+mod commands;
+mod environments;
+mod metadata;
+mod parser;
+mod utilities;
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::{ExtractionResult, Metadata, Table};
+use async_trait::async_trait;
+
+use parser::LatexParser;
+
+/// LaTeX document extractor
+pub struct LatexExtractor;
+
+impl LatexExtractor {
+    /// Create a new LaTeX extractor.
+    pub fn new() -> Self {
+        Self
+    }
+
+    /// Parse LaTeX content and extract text.
+    fn extract_from_latex(content: &str) -> (String, Metadata, Vec<Table>) {
+        let mut parser = LatexParser::new(content);
+        parser.parse()
+    }
+}
+
+impl Default for LatexExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Plugin for LatexExtractor {
+    fn name(&self) -> &str {
+        "latex-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn description(&self) -> &str {
+        "Native Rust LaTeX document extractor with metadata and table support"
+    }
+
+    fn author(&self) -> &str {
+        "Kreuzberg Team"
+    }
+}
+
+#[async_trait]
+impl DocumentExtractor for LatexExtractor {
+    #[cfg_attr(feature = "otel", tracing::instrument(
+        skip(self, content, _config),
+        fields(
+            extractor.name = self.name(),
+            content.size_bytes = content.len(),
+        )
+    ))]
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let latex_str = String::from_utf8_lossy(content).to_string();
+        let (text, metadata, tables) = Self::extract_from_latex(&latex_str);
+
+        Ok(ExtractionResult {
+            content: text,
+            mime_type: mime_type.to_string(),
+            metadata,
+            tables,
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            djot_content: None,
+            pages: None,
+            elements: None,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &["application/x-latex", "text/x-tex"]
+    }
+
+    fn priority(&self) -> i32 {
+        50
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_basic_title_extraction() {
+        let latex = r#"\title{Hello World}"#;
+        let (_, metadata, _) = LatexExtractor::extract_from_latex(latex);
+        assert_eq!(
+            metadata.additional.get("title").and_then(|v| v.as_str()),
+            Some("Hello World")
+        );
+    }
+
+    #[test]
+    fn test_author_extraction() {
+        let latex = r#"\author{John Doe}"#;
+        let (_, metadata, _) = LatexExtractor::extract_from_latex(latex);
+        assert!(metadata.additional.contains_key("author"));
+    }
+
+    #[test]
+    fn test_section_extraction() {
+        let latex = r#"\begin{document}\section{Introduction}\end{document}"#;
+        let (content, _, _) = LatexExtractor::extract_from_latex(latex);
+        assert!(content.contains("Introduction"));
+    }
+}

data/vendor/kreuzberg/src/extractors/latex/parser.rs

@@ -0,0 +1,231 @@
+//! Core LaTeX parser implementation.
+//!
+//! This module contains the main LatexParser struct and the core parsing logic
+//! that orchestrates document structure extraction.
+
+use super::commands::process_line;
+use super::environments::{process_list, process_table, process_table_with_caption};
+use super::metadata::extract_metadata_from_line;
+use super::utilities::{collect_environment, extract_braced, extract_env_name};
+use crate::types::{Metadata, Table};
+
+/// LaTeX parser state machine.
+///
+/// Maintains parsing state including metadata, tables, and output as it
+/// processes a LaTeX document line by line.
+pub struct LatexParser<'a> {
+    source: &'a str,
+    metadata: Metadata,
+    tables: Vec<Table>,
+    output: String,
+}
+
+impl<'a> LatexParser<'a> {
+    /// Creates a new LaTeX parser for the given source.
+    pub fn new(source: &'a str) -> Self {
+        Self {
+            source,
+            metadata: Metadata::default(),
+            tables: Vec::new(),
+            output: String::new(),
+        }
+    }
+
+    /// Parses the LaTeX document and returns extracted content, metadata, and tables.
+    pub fn parse(&mut self) -> (String, Metadata, Vec<Table>) {
+        let lines: Vec<&str> = self.source.lines().collect();
+        let mut in_document = false;
+        let mut skip_until_end = None::<String>;
+        let mut i = 0;
+
+        // Detect plain TeX documents (no \begin{document})
+        let is_plain_tex = self.source.contains("\\bye") && !self.source.contains("\\begin{document}");
+        if is_plain_tex {
+            in_document = true;
+        }
+
+        while i < lines.len() {
+            let line = lines[i];
+            let trimmed = line.trim();
+
+            // Handle environments we're skipping
+            if let Some(ref env) = skip_until_end {
+                if trimmed.contains(&format!("\\end{{{}}}", env)) {
+                    skip_until_end = None;
+                }
+                i += 1;
+                continue;
+            }
+
+            // Handle plain TeX end marker
+            if is_plain_tex && trimmed.contains("\\bye") {
+                break;
+            }
+
+            // Extract metadata from preamble
+            if !in_document && !is_plain_tex {
+                extract_metadata_from_line(trimmed, &mut self.metadata);
+            }
+
+            // Handle \begin{document}
+            if !is_plain_tex && trimmed.contains("\\begin{document}") {
+                in_document = true;
+
+                // Handle single-line documents
+                if trimmed.contains("\\end{document}") {
+                    self.process_single_line_document(trimmed);
+                    break;
+                }
+
+                i += 1;
+                continue;
+            }
+
+            // Handle \end{document}
+            if !is_plain_tex && trimmed.contains("\\end{document}") {
+                break;
+            }
+
+            // Process document content
+            if in_document {
+                if self.process_environments(&lines, trimmed, &mut i, &mut skip_until_end) {
+                    continue;
+                }
+
+                self.process_sections_and_content(trimmed, &lines, &mut i);
+            }
+
+            i += 1;
+        }
+
+        let content = self.output.trim().to_string();
+        (content, self.metadata.clone(), self.tables.clone())
+    }
+
+    /// Processes a single-line document (both \begin and \end on same line).
+    fn process_single_line_document(&mut self, trimmed: &str) {
+        let Some(begin_pos) = trimmed.find("\\begin{document}") else {
+            return;
+        };
+        let Some(end_pos) = trimmed.find("\\end{document}") else {
+            return;
+        };
+        let content_between = trimmed[begin_pos + 16..end_pos].trim();
+        if !content_between.is_empty() {
+            if content_between.starts_with("\\section{") {
+                if let Some(title) = extract_braced(content_between, "section") {
+                    self.output.push_str(&format!("\n# {}\n\n", title));
+                }
+            } else {
+                let processed = process_line(content_between);
+                if !processed.is_empty() {
+                    self.output.push_str(&processed);
+                    self.output.push('\n');
+                }
+            }
+        }
+    }
+
+    /// Processes LaTeX environments (lists, tables, math).
+    ///
+    /// Returns true if an environment was processed and the line index was updated.
+    fn process_environments(
+        &mut self,
+        lines: &[&str],
+        trimmed: &str,
+        i: &mut usize,
+        skip_until_end: &mut Option<String>,
+    ) -> bool {
+        if !trimmed.contains("\\begin{") {
+            return false;
+        }
+
+        let Some(env_name) = extract_env_name(trimmed) else {
+            return false;
+        };
+
+        match env_name.as_str() {
+            "itemize" | "enumerate" | "description" => {
+                let (env_content, new_i) = collect_environment(lines, *i, &env_name);
+                process_list(&env_content, &env_name, &mut self.output);
+                *i = new_i;
+                true
+            }
+            "tabular" => {
+                let (env_content, new_i) = collect_environment(lines, *i, "tabular");
+                process_table(&env_content, &mut self.output, &mut self.tables);
+                *i = new_i;
+                true
+            }
+            "table" => {
+                let (env_content, new_i) = collect_environment(lines, *i, "table");
+                process_table_with_caption(&env_content, &mut self.output, &mut self.tables);
+                *i = new_i;
+                true
+            }
+            "equation" | "align" | "gather" | "multline" => {
+                let (env_content, new_i) = collect_environment(lines, *i, &env_name);
+                self.output.push_str("$$\\begin{");
+                self.output.push_str(&env_name);
+                self.output.push_str("}\n");
+                self.output.push_str(&env_content);
+                self.output.push_str("\\end{");
+                self.output.push_str(&env_name);
+                self.output.push_str("}$$\n\n");
+                *i = new_i;
+                true
+            }
+            _ => {
+                *skip_until_end = Some(env_name);
+                false
+            }
+        }
+    }
+
+    /// Processes section headings, display math, and regular content.
+    fn process_sections_and_content(&mut self, trimmed: &str, lines: &[&str], i: &mut usize) {
+        if trimmed.starts_with("\\section{") {
+            if let Some(title) = extract_braced(trimmed, "section") {
+                self.output.push_str(&format!("\n# {}\n\n", title));
+            }
+        } else if trimmed.starts_with("\\subsection{") {
+            if let Some(title) = extract_braced(trimmed, "subsection") {
+                self.output.push_str(&format!("## {}\n\n", title));
+            }
+        } else if trimmed.starts_with("\\subsubsection{") {
+            if let Some(title) = extract_braced(trimmed, "subsubsection") {
+                self.output.push_str(&format!("### {}\n\n", title));
+            }
+        } else if trimmed.starts_with("\\[") {
+            // Display math mode
+            self.process_display_math(trimmed, lines, i);
+        } else if !trimmed.is_empty() && !trimmed.starts_with("%") {
+            // Regular content
+            let processed = process_line(trimmed);
+            if !processed.is_empty() {
+                self.output.push_str(&processed);
+                self.output.push('\n');
+            }
+        }
+    }
+
+    /// Processes display math mode \[...\].
+    fn process_display_math(&mut self, trimmed: &str, lines: &[&str], i: &mut usize) {
+        let mut math_content = trimmed.to_string();
+        if !trimmed.contains("\\]") {
+            // Math spans multiple lines
+            *i += 1;
+            while *i < lines.len() {
+                let math_line = lines[*i];
+                math_content.push('\n');
+                math_content.push_str(math_line);
+                if math_line.trim().contains("\\]") {
+                    break;
+                }
+                *i += 1;
+            }
+        }
+        self.output.push_str(&math_content);
+        self.output.push('\n');
+    }
+}

data/vendor/kreuzberg/src/extractors/latex/utilities.rs

@@ -0,0 +1,126 @@
+//! Utility functions for LaTeX parsing.
+//!
+//! This module contains helper functions for text cleaning, brace extraction,
+//! and other common operations used throughout the LaTeX parser.
+
+/// Extracts content from within braces for a given command.
+///
+/// Example: `\title{Hello World}` with command "title" returns "Hello World"
+pub fn extract_braced(text: &str, command: &str) -> Option<String> {
+    let pattern = format!("\\{}{{", command);
+    if let Some(start) = text.find(&pattern) {
+        let after = &text[start + pattern.len()..];
+        let mut depth = 1;
+        let mut content = String::new();
+
+        for ch in after.chars() {
+            match ch {
+                '{' => {
+                    depth += 1;
+                    content.push(ch);
+                }
+                '}' => {
+                    depth -= 1;
+                    if depth == 0 {
+                        return Some(clean_text(&content));
+                    }
+                    content.push(ch);
+                }
+                _ => content.push(ch),
+            }
+        }
+    }
+    None
+}
+
+/// Reads braced content from a character iterator.
+///
+/// Handles nested braces correctly and maintains proper depth tracking.
+pub fn read_braced_from_chars(chars: &mut std::iter::Peekable<std::str::Chars>) -> Option<String> {
+    // Skip whitespace before opening brace
+    while let Some(&c) = chars.peek() {
+        if c.is_whitespace() {
+            chars.next();
+        } else {
+            break;
+        }
+    }
+
+    // Check for opening brace
+    if chars.peek() != Some(&'{') {
+        return None;
+    }
+    chars.next(); // Consume '{'
+
+    let mut content = String::new();
+    let mut depth = 1;
+
+    for c in chars.by_ref() {
+        match c {
+            '{' => {
+                depth += 1;
+                content.push(c);
+            }
+            '}' => {
+                depth -= 1;
+                if depth == 0 {
+                    return Some(content);
+                }
+                content.push(c);
+            }
+            _ => content.push(c),
+        }
+    }
+
+    Some(content)
+}
+
+/// Extracts environment name from a \begin{} statement.
+///
+/// Example: `\begin{itemize}` returns "itemize"
+pub fn extract_env_name(line: &str) -> Option<String> {
+    if let Some(start) = line.find("\\begin{") {
+        let after = &line[start + 7..];
+        if let Some(end) = after.find('}') {
+            return Some(after[..end].to_string());
+        }
+    }
+    None
+}
+
+/// Cleans LaTeX text by removing escape sequences.
+///
+/// Handles common LaTeX escape sequences like \\&, \\#, \\\_, etc.
+pub fn clean_text(text: &str) -> String {
+    text.to_string()
+        .replace("\\\\", "\n")
+        .replace("\\&", "&")
+        .replace("\\#", "#")
+        .replace("\\_", "_")
+        .replace("\\{", "{")
+        .replace("\\}", "}")
+        .replace("\\%", "%")
+        .trim()
+        .to_string()
+}
+
+/// Collects content of an environment from begin to end.
+///
+/// Returns the content and the index of the line after \end{environment}.
+pub fn collect_environment(lines: &[&str], start_idx: usize, env_name: &str) -> (String, usize) {
+    let mut content = String::new();
+    let mut i = start_idx + 1;
+    let end_marker = format!("\\end{{{}}}", env_name);
+
+    while i < lines.len() {
+        let line = lines[i];
+        if line.trim().contains(&end_marker) {
+            return (content, i + 1);
+        }
+        content.push_str(line);
+        content.push('\n');
+        i += 1;
+    }
+
+    (content, i)
+}