kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/extractors/rtf/parser.rs

@@ -0,0 +1,259 @@
+//! Core RTF parsing logic.
+
+use crate::extractors::rtf::encoding::{decode_windows_1252, parse_hex_byte, parse_rtf_control_word};
+use crate::extractors::rtf::formatting::normalize_whitespace;
+use crate::extractors::rtf::images::extract_image_metadata;
+use crate::extractors::rtf::tables::TableState;
+use crate::types::Table;
+
+/// Extract text and image metadata from RTF document.
+///
+/// This function extracts plain text from an RTF document by:
+/// 1. Tokenizing control sequences and text
+/// 2. Converting encoded characters to Unicode
+/// 3. Extracting text while skipping formatting groups
+/// 4. Detecting and extracting image metadata (\pict sections)
+/// 5. Normalizing whitespace
+pub fn extract_text_from_rtf(content: &str) -> (String, Vec<Table>) {
+    let mut result = String::new();
+    let mut chars = content.chars().peekable();
+    let mut tables: Vec<Table> = Vec::new();
+    let mut table_state: Option<TableState> = None;
+
+    let ensure_table = |table_state: &mut Option<TableState>| {
+        if table_state.is_none() {
+            *table_state = Some(TableState::new());
+        }
+    };
+
+    let finalize_table = |state_opt: &mut Option<TableState>, tables: &mut Vec<Table>| {
+        if let Some(state) = state_opt.take()
+            && let Some(table) = state.finalize()
+        {
+            tables.push(table);
+        }
+    };
+
+    while let Some(ch) = chars.next() {
+        match ch {
+            '\\' => {
+                if let Some(&next_ch) = chars.peek() {
+                    match next_ch {
+                        '\\' | '{' | '}' => {
+                            chars.next();
+                            result.push(next_ch);
+                        }
+                        '\'' => {
+                            chars.next();
+                            let hex1 = chars.next();
+                            let hex2 = chars.next();
+                            if let (Some(h1), Some(h2)) = (hex1, hex2)
+                                && let Some(byte) = parse_hex_byte(h1, h2)
+                            {
+                                let decoded = decode_windows_1252(byte);
+                                result.push(decoded);
+                                if let Some(state) = table_state.as_mut()
+                                    && state.in_row
+                                {
+                                    state.current_cell.push(decoded);
+                                }
+                            }
+                        }
+                        'u' => {
+                            chars.next();
+                            let mut num_str = String::new();
+                            while let Some(&c) = chars.peek() {
+                                if c.is_ascii_digit() || c == '-' {
+                                    num_str.push(c);
+                                    chars.next();
+                                } else {
+                                    break;
+                                }
+                            }
+                            if let Ok(code_num) = num_str.parse::<i32>() {
+                                let code_u = if code_num < 0 {
+                                    (code_num + 65536) as u32
+                                } else {
+                                    code_num as u32
+                                };
+                                if let Some(c) = char::from_u32(code_u) {
+                                    result.push(c);
+                                    if let Some(state) = table_state.as_mut()
+                                        && state.in_row
+                                    {
+                                        state.current_cell.push(c);
+                                    }
+                                }
+                            }
+                        }
+                        _ => {
+                            let (control_word, _) = parse_rtf_control_word(&mut chars);
+                            handle_control_word(
+                                &control_word,
+                                &mut chars,
+                                &mut result,
+                                &mut table_state,
+                                &mut tables,
+                                &ensure_table,
+                                &finalize_table,
+                            );
+                        }
+                    }
+                }
+            }
+            '{' | '}' => {
+                if !result.is_empty() && !result.ends_with(' ') {
+                    result.push(' ');
+                }
+            }
+            ' ' | '\t' | '\n' | '\r' => {
+                if !result.is_empty() && !result.ends_with(' ') {
+                    result.push(' ');
+                }
+                if let Some(state) = table_state.as_mut()
+                    && state.in_row
+                    && !state.current_cell.ends_with(' ')
+                {
+                    state.current_cell.push(' ');
+                }
+            }
+            _ => {
+                if let Some(state) = table_state.as_ref()
+                    && !state.in_row
+                    && !state.rows.is_empty()
+                {
+                    finalize_table(&mut table_state, &mut tables);
+                }
+                result.push(ch);
+                if let Some(state) = table_state.as_mut()
+                    && state.in_row
+                {
+                    state.current_cell.push(ch);
+                }
+            }
+        }
+    }
+
+    if table_state.is_some() {
+        finalize_table(&mut table_state, &mut tables);
+    }
+
+    (normalize_whitespace(&result), tables)
+}
+
+/// Handle an RTF control word during parsing.
+#[allow(clippy::too_many_arguments)]
+fn handle_control_word(
+    control_word: &str,
+    chars: &mut std::iter::Peekable<std::str::Chars>,
+    result: &mut String,
+    table_state: &mut Option<TableState>,
+    tables: &mut Vec<Table>,
+    ensure_table: &dyn Fn(&mut Option<TableState>),
+    finalize_table: &dyn Fn(&mut Option<TableState>, &mut Vec<Table>),
+) {
+    match control_word {
+        "pict" => {
+            let image_metadata = extract_image_metadata(chars);
+            if !image_metadata.is_empty() {
+                result.push('!');
+                result.push('[');
+                result.push_str("image");
+                result.push(']');
+                result.push('(');
+                result.push_str(&image_metadata);
+                result.push(')');
+                result.push(' ');
+                if let Some(state) = table_state.as_mut()
+                    && state.in_row
+                {
+                    state.current_cell.push('!');
+                    state.current_cell.push('[');
+                    state.current_cell.push_str("image");
+                    state.current_cell.push(']');
+                    state.current_cell.push('(');
+                    state.current_cell.push_str(&image_metadata);
+                    state.current_cell.push(')');
+                    state.current_cell.push(' ');
+                }
+            }
+        }
+        "par" => {
+            if table_state.is_some() {
+                finalize_table(table_state, tables);
+            }
+            if !result.is_empty() && !result.ends_with('\n') {
+                result.push('\n');
+                result.push('\n');
+            }
+        }
+        "tab" => {
+            result.push('\t');
+            if let Some(state) = table_state.as_mut()
+                && state.in_row
+            {
+                state.current_cell.push('\t');
+            }
+        }
+        "bullet" => {
+            result.push('•');
+        }
+        "lquote" => {
+            result.push('\u{2018}');
+        }
+        "rquote" => {
+            result.push('\u{2019}');
+        }
+        "ldblquote" => {
+            result.push('\u{201C}');
+        }
+        "rdblquote" => {
+            result.push('\u{201D}');
+        }
+        "endash" => {
+            result.push('\u{2013}');
+        }
+        "emdash" => {
+            result.push('\u{2014}');
+        }
+        "trowd" => {
+            ensure_table(table_state);
+            if let Some(state) = table_state.as_mut() {
+                state.start_row();
+            }
+            if !result.is_empty() && !result.ends_with('\n') {
+                result.push('\n');
+            }
+            if !result.ends_with('|') {
+                result.push('|');
+                result.push(' ');
+            }
+        }
+        "cell" => {
+            if !result.ends_with('|') {
+                if !result.ends_with(' ') && !result.is_empty() {
+                    result.push(' ');
+                }
+                result.push('|');
+            }
+            if !result.ends_with(' ') {
+                result.push(' ');
+            }
+        }
+        "row" => {
+            ensure_table(table_state);
+            if let Some(state) = table_state.as_mut()
+                && (state.in_row || !state.current_cell.is_empty())
+            {
+                state.push_row();
+            }
+            if !result.ends_with('|') {
+                result.push('|');
+            }
+            if !result.ends_with('\n') {
+                result.push('\n');
+            }
+        }
+        _ => {}
+    }
+}

data/vendor/kreuzberg/src/extractors/rtf/tables.rs

@@ -0,0 +1,83 @@
+//! Table extraction and state management for RTF documents.
+
+use crate::extraction::cells_to_markdown;
+use crate::types::Table;
+
+/// State machine for tracking table construction during RTF parsing.
+pub struct TableState {
+    pub rows: Vec<Vec<String>>,
+    pub current_row: Vec<String>,
+    pub current_cell: String,
+    pub in_row: bool,
+}
+
+impl TableState {
+    /// Create a new empty table state.
+    pub fn new() -> Self {
+        Self {
+            rows: Vec::new(),
+            current_row: Vec::new(),
+            current_cell: String::new(),
+            in_row: false,
+        }
+    }
+
+    /// Push the current cell content to the current row.
+    pub fn push_cell(&mut self) {
+        let cell = self.current_cell.trim().to_string();
+        self.current_row.push(cell);
+        self.current_cell.clear();
+    }
+
+    /// Push the current row to the rows collection.
+    pub fn push_row(&mut self) {
+        if self.in_row || !self.current_cell.is_empty() {
+            self.push_cell();
+            self.in_row = false;
+        }
+        if !self.current_row.is_empty() {
+            self.rows.push(self.current_row.clone());
+            self.current_row.clear();
+        }
+    }
+
+    /// Start a new table row.
+    pub fn start_row(&mut self) {
+        if self.in_row {
+            self.push_row();
+        }
+        self.in_row = true;
+        self.current_cell.clear();
+        self.current_row.clear();
+    }
+
+    /// Check if this table has any content.
+    #[allow(dead_code)]
+    pub fn is_empty(&self) -> bool {
+        self.rows.is_empty() && self.current_row.is_empty() && self.current_cell.is_empty()
+    }
+
+    /// Finalize the table and convert it to a Table struct.
+    pub fn finalize(mut self) -> Option<Table> {
+        if self.in_row || !self.current_cell.is_empty() || !self.current_row.is_empty() {
+            self.push_row();
+        }
+
+        if self.rows.is_empty() {
+            return None;
+        }
+
+        let markdown = cells_to_markdown(&self.rows);
+        Some(Table {
+            cells: self.rows,
+            markdown,
+            page_number: 1,
+        })
+    }
+}
+
+impl Default for TableState {
+    fn default() -> Self {
+        Self::new()
+    }
+}

data/vendor/kreuzberg/src/extractors/structured.rs

@@ -86,6 +86,8 @@ impl DocumentExtractor for StructuredExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            elements: None,
+            djot_content: None,
         })
     }
 

data/vendor/kreuzberg/src/extractors/text.rs

@@ -91,6 +91,8 @@ impl DocumentExtractor for PlainTextExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            elements: None,
+            djot_content: None,
         })
     }
 
@@ -184,6 +186,8 @@ impl DocumentExtractor for MarkdownExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            elements: None,
+            djot_content: None,
         })
     }
 

data/vendor/kreuzberg/src/extractors/typst.rs

@@ -112,7 +112,9 @@ impl DocumentExtractor for TypstExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         })
     }
 

data/vendor/kreuzberg/src/extractors/xml.rs

@@ -71,6 +71,8 @@ impl SyncExtractor for XmlExtractor {
             chunks: None,
             images: None,
             pages: None,
+            djot_content: None,
+            elements: None,
         })
     }
 }

data/vendor/kreuzberg/src/keywords/processor.rs

@@ -114,6 +114,8 @@ machine learning that uses neural networks with multiple layers.
             chunks: None,
             images: None,
             pages: None,
+            elements: None,
+            djot_content: None,
         };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -143,6 +145,8 @@ machine learning that uses neural networks with multiple layers.
             chunks: None,
             images: None,
             pages: None,
+            elements: None,
+            djot_content: None,
         };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -168,6 +172,8 @@ machine learning that uses neural networks with multiple layers.
             chunks: None,
             images: None,
             pages: None,
+            elements: None,
+            djot_content: None,
         };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -193,6 +199,8 @@ machine learning that uses neural networks with multiple layers.
             chunks: None,
             images: None,
             pages: None,
+            elements: None,
+            djot_content: None,
         };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -229,6 +237,8 @@ machine learning that uses neural networks with multiple layers.
             chunks: None,
             images: None,
             pages: None,
+            elements: None,
+            djot_content: None,
         };
 
         let config_with_keywords = ExtractionConfig {
@@ -254,6 +264,8 @@ machine learning that uses neural networks with multiple layers.
             chunks: None,
             images: None,
             pages: None,
+            elements: None,
+            djot_content: None,
         };
 
         let long_result = ExtractionResult {
@@ -265,6 +277,8 @@ machine learning that uses neural networks with multiple layers.
             chunks: None,
             images: None,
             pages: None,
+            elements: None,
+            djot_content: None,
         };
 
         let short_duration = processor.estimated_duration_ms(&short_result);

data/vendor/kreuzberg/src/language_detection/processor.rs

@@ -108,7 +108,9 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -132,7 +134,9 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -167,7 +171,9 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         };
 
         let config_with_lang = ExtractionConfig {
@@ -196,7 +202,9 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         };
 
         let long_result = ExtractionResult {
@@ -207,7 +215,9 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         };
 
         let short_duration = processor.estimated_duration_ms(&short_result);

data/vendor/kreuzberg/src/lib.rs

@@ -89,14 +89,14 @@ pub use core::extractor::{batch_extract_file_sync, extract_file_sync};
 
 pub use core::config::{
     ChunkingConfig, EmbeddingConfig, EmbeddingModelType, ExtractionConfig, ImageExtractionConfig,
-    LanguageDetectionConfig, OcrConfig, PostProcessorConfig, TokenReductionConfig,
+    LanguageDetectionConfig, OcrConfig, OutputFormat, PageConfig, PostProcessorConfig, TokenReductionConfig,
 };
 
 #[cfg(feature = "api")]
 pub use core::server_config::ServerConfig;
 
 #[cfg(feature = "pdf")]
-pub use core::config::PdfConfig;
+pub use core::config::{HierarchyConfig, PdfConfig};
 
 pub use core::mime::{
     DOCX_MIME_TYPE, EXCEL_MIME_TYPE, HTML_MIME_TYPE, JSON_MIME_TYPE, MARKDOWN_MIME_TYPE, PDF_MIME_TYPE,