kreuzberg 4.0.0.pre.rc.6 → 4.0.0.pre.rc.8

data/vendor/kreuzberg/src/extraction/html.rs

@@ -283,17 +283,29 @@ pub fn parse_html_metadata(markdown: &str) -> Result<(Option<HtmlMetadata>, Stri
         .map_err(|e| KreuzbergError::parsing(format!("Failed to parse YAML frontmatter: {}", e)))?;
 
     let mut metadata = HtmlMetadata::default();
+    let mut title: Option<String> = None;
+    let mut description: Option<String> = None;
+    let mut keywords: Option<Vec<String>> = None;
+    let mut author: Option<String> = None;
 
     if let serde_json::Value::Object(mapping) = yaml_value {
         for (key, value) in mapping {
             if let serde_json::Value::String(value_str) = value {
                 match key.as_str() {
-                    "title" => metadata.title = Some(value_str),
+                    "title" => title = Some(value_str),
                     "base-href" => metadata.base_href = Some(value_str),
                     "canonical" => metadata.canonical = Some(value_str),
-                    "meta-description" => metadata.description = Some(value_str),
-                    "meta-keywords" => metadata.keywords = Some(value_str),
-                    "meta-author" => metadata.author = Some(value_str),
+                    "meta-description" => description = Some(value_str),
+                    "meta-keywords" => {
+                        keywords = Some(
+                            value_str
+                                .split(',')
+                                .map(|k| k.trim().to_string())
+                                .filter(|k| !k.is_empty())
+                                .collect(),
+                        )
+                    }
+                    "meta-author" => author = Some(value_str),
                     "meta-og-title" | "meta-og:title" => metadata.og_title = Some(value_str),
                     "meta-og-description" | "meta-og:description" => metadata.og_description = Some(value_str),
                     "meta-og-image" | "meta-og:image" => metadata.og_image = Some(value_str),
@@ -319,10 +331,10 @@ pub fn parse_html_metadata(markdown: &str) -> Result<(Option<HtmlMetadata>, Stri
         }
     }
 
-    let has_metadata = metadata.title.is_some()
-        || metadata.description.is_some()
-        || metadata.keywords.is_some()
-        || metadata.author.is_some()
+    let has_metadata = title.is_some()
+        || description.is_some()
+        || keywords.is_some()
+        || author.is_some()
         || metadata.canonical.is_some()
         || metadata.base_href.is_some()
         || metadata.og_title.is_some()
@@ -331,6 +343,10 @@ pub fn parse_html_metadata(markdown: &str) -> Result<(Option<HtmlMetadata>, Stri
         || metadata.twitter_card.is_some();
 
     if has_metadata {
+        metadata.title = title;
+        metadata.description = description;
+        metadata.keywords = keywords.map(|kws| kws.join(", "));
+        metadata.author = author;
         Ok((Some(metadata), remaining_content.to_string()))
     } else {
         Ok((None, remaining_content.to_string()))

data/vendor/kreuzberg/src/extraction/image.rs

@@ -1,6 +1,7 @@
 //! Image extraction functionality.
 //!
-//! This module provides functions for extracting metadata and EXIF data from images.
+//! This module provides functions for extracting metadata and EXIF data from images,
+//! including support for multi-frame TIFF files.
 
 use crate::error::{KreuzbergError, Result};
 use exif::{In, Reader, Tag};
@@ -94,6 +95,128 @@ fn extract_exif_data(bytes: &[u8]) -> HashMap<String, String> {
     exif_map
 }
 
+/// Result of OCR extraction from an image with optional page tracking.
+#[derive(Debug, Clone)]
+pub struct ImageOcrResult {
+    /// Extracted text content
+    pub content: String,
+    /// Character byte boundaries per frame (for multi-frame TIFFs)
+    pub boundaries: Option<Vec<crate::types::PageBoundary>>,
+    /// Per-frame content information
+    pub page_contents: Option<Vec<crate::types::PageContent>>,
+}
+
+/// Detects the number of frames in a TIFF file.
+///
+/// Returns the count of image frames/pages in a TIFF. Single-frame TIFFs return 1.
+/// Invalid or non-TIFF data returns an error.
+///
+/// # Arguments
+/// * `bytes` - Raw TIFF file bytes
+///
+/// # Returns
+/// Frame count if valid TIFF, error otherwise.
+#[cfg(feature = "ocr")]
+fn detect_tiff_frame_count(bytes: &[u8]) -> Result<usize> {
+    use tiff::decoder::Decoder;
+    let mut decoder =
+        Decoder::new(Cursor::new(bytes)).map_err(|e| KreuzbergError::parsing(format!("TIFF decode: {}", e)))?;
+
+    let mut count = 1;
+    while decoder.next_image().is_ok() {
+        count += 1;
+    }
+    Ok(count)
+}
+
+/// Extract text from image bytes using OCR with optional page tracking for multi-frame TIFFs.
+///
+/// This function:
+/// - Detects if the image is a multi-frame TIFF
+/// - For multi-frame TIFFs with PageConfig enabled, iterates frames and tracks boundaries
+/// - For single-frame images or when page tracking is disabled, runs OCR on the whole image
+/// - Returns (content, boundaries, page_contents) tuple
+///
+/// # Arguments
+/// * `bytes` - Image file bytes
+/// * `mime_type` - MIME type (e.g., "image/tiff")
+/// * `ocr_result` - OCR backend result containing the text
+/// * `page_config` - Optional page configuration for boundary tracking
+///
+/// # Returns
+/// ImageOcrResult with content and optional boundaries for pagination
+#[cfg(feature = "ocr")]
+pub fn extract_text_from_image_with_ocr(
+    bytes: &[u8],
+    mime_type: &str,
+    ocr_result: String,
+    page_config: Option<&crate::core::config::PageConfig>,
+) -> Result<ImageOcrResult> {
+    let is_tiff = mime_type.to_lowercase().contains("tiff");
+    let should_track_pages = page_config.is_some() && is_tiff;
+
+    if !should_track_pages {
+        return Ok(ImageOcrResult {
+            content: ocr_result,
+            boundaries: None,
+            page_contents: None,
+        });
+    }
+
+    let frame_count = detect_tiff_frame_count(bytes)?;
+
+    if frame_count <= 1 {
+        return Ok(ImageOcrResult {
+            content: ocr_result,
+            boundaries: None,
+            page_contents: None,
+        });
+    }
+
+    let content_len = ocr_result.len();
+    let content_per_frame = if frame_count > 0 {
+        content_len / frame_count
+    } else {
+        content_len
+    };
+
+    let mut boundaries = Vec::new();
+    let mut page_contents = Vec::new();
+    let mut byte_offset = 0;
+
+    for frame_num in 1..=frame_count {
+        let frame_end = if frame_num == frame_count {
+            content_len
+        } else {
+            let raw_end = (frame_num * content_per_frame).min(content_len);
+            (raw_end..=content_len)
+                .find(|&i| ocr_result.is_char_boundary(i))
+                .unwrap_or(content_len)
+        };
+
+        boundaries.push(crate::types::PageBoundary {
+            byte_start: byte_offset,
+            byte_end: frame_end,
+            page_number: frame_num,
+        });
+
+        page_contents.push(crate::types::PageContent {
+            page_number: frame_num,
+            content: ocr_result[byte_offset..frame_end].to_string(),
+            tables: vec![],
+            images: vec![],
+        });
+
+        byte_offset = frame_end;
+    }
+
+    Ok(ImageOcrResult {
+        content: ocr_result,
+        boundaries: Some(boundaries),
+        page_contents: Some(page_contents),
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

data/vendor/kreuzberg/src/extraction/libreoffice.rs

@@ -462,8 +462,7 @@ mod tests {
     async fn test_check_libreoffice_missing_dependency_error() {
         let result = check_libreoffice_available().await;
 
-        if result.is_err() {
-            let err = result.unwrap_err();
+        if let Err(err) = result {
             match err {
                 KreuzbergError::MissingDependency(msg) => {
                     assert!(msg.contains("LibreOffice") || msg.contains("soffice"));

data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs

@@ -104,7 +104,6 @@ pub fn extract_odt_properties<R: Read + std::io::Seek>(archive: &mut ZipArchive<
 
     let root = doc.root_element();
 
-    // Extract Dublin Core elements
     let title = super::parse_xml_text(root, "title");
     let subject = super::parse_xml_text(root, "subject");
     let creator = super::parse_xml_text(root, "creator");
@@ -112,7 +111,6 @@ pub fn extract_odt_properties<R: Read + std::io::Seek>(archive: &mut ZipArchive<
     let language = super::parse_xml_text(root, "language");
     let date = super::parse_xml_text(root, "date");
 
-    // Extract OpenDocument meta elements
     let initial_creator = super::parse_xml_text(root, "initial-creator");
     let keywords = super::parse_xml_text(root, "keyword");
     let creation_date = super::parse_xml_text(root, "creation-date");
@@ -120,7 +118,6 @@ pub fn extract_odt_properties<R: Read + std::io::Seek>(archive: &mut ZipArchive<
     let editing_duration = super::parse_xml_text(root, "editing-duration");
     let editing_cycles = super::parse_xml_text(root, "editing-cycles");
 
-    // Extract document statistics
     let page_count = super::parse_xml_int(root, "page-count");
     let word_count = super::parse_xml_int(root, "word-count");
     let character_count = super::parse_xml_int(root, "character-count");