kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/extractors/epub/parsing.rs

@@ -0,0 +1,86 @@
+//! EPUB ZIP archive and XML parsing utilities.
+//!
+//! Provides low-level parsing functionality for EPUB container structure,
+//! including ZIP archive operations and container.xml parsing.
+
+use crate::Result;
+use roxmltree;
+use std::io::Cursor;
+use zip::ZipArchive;
+
+/// Parse container.xml to find the OPF file path
+pub(super) fn parse_container_xml(xml: &str) -> Result<String> {
+    match roxmltree::Document::parse(xml) {
+        Ok(doc) => {
+            for node in doc.descendants() {
+                if node.tag_name().name() == "rootfile"
+                    && let Some(full_path) = node.attribute("full-path")
+                {
+                    return Ok(full_path.to_string());
+                }
+            }
+            Err(crate::KreuzbergError::Parsing {
+                message: "No rootfile found in container.xml".to_string(),
+                source: None,
+            })
+        }
+        Err(e) => Err(crate::KreuzbergError::Parsing {
+            message: format!("Failed to parse container.xml: {}", e),
+            source: None,
+        }),
+    }
+}
+
+/// Read a file from the ZIP archive
+pub(super) fn read_file_from_zip(archive: &mut ZipArchive<Cursor<Vec<u8>>>, path: &str) -> Result<String> {
+    match archive.by_name(path) {
+        Ok(mut file) => {
+            let mut content = String::new();
+            match std::io::Read::read_to_string(&mut file, &mut content) {
+                Ok(_) => Ok(content),
+                Err(e) => Err(crate::KreuzbergError::Parsing {
+                    message: format!("Failed to read file from EPUB: {}", e),
+                    source: None,
+                }),
+            }
+        }
+        Err(e) => Err(crate::KreuzbergError::Parsing {
+            message: format!("File not found in EPUB: {} ({})", path, e),
+            source: None,
+        }),
+    }
+}
+
+/// Resolve a relative path within the manifest directory
+pub(super) fn resolve_path(base_dir: &str, relative_path: &str) -> String {
+    if relative_path.starts_with('/') {
+        relative_path.trim_start_matches('/').to_string()
+    } else if base_dir.is_empty() || base_dir == "." {
+        relative_path.to_string()
+    } else {
+        format!("{}/{}", base_dir.trim_end_matches('/'), relative_path)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_resolve_path_with_base_dir() {
+        let result = resolve_path("OEBPS", "chapter.xhtml");
+        assert_eq!(result, "OEBPS/chapter.xhtml");
+    }
+
+    #[test]
+    fn test_resolve_path_absolute() {
+        let result = resolve_path("OEBPS", "/chapter.xhtml");
+        assert_eq!(result, "chapter.xhtml");
+    }
+
+    #[test]
+    fn test_resolve_path_empty_base() {
+        let result = resolve_path("", "chapter.xhtml");
+        assert_eq!(result, "chapter.xhtml");
+    }
+}

data/vendor/kreuzberg/src/extractors/excel.rs

@@ -140,6 +140,8 @@ impl DocumentExtractor for ExcelExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
+            elements: None,
         })
     }
 
@@ -184,6 +186,8 @@ impl DocumentExtractor for ExcelExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
+            elements: None,
         })
     }
 

data/vendor/kreuzberg/src/extractors/fictionbook.rs

@@ -440,7 +440,9 @@ impl DocumentExtractor for FictionBookExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         })
     }
 

data/vendor/kreuzberg/src/extractors/frontmatter_utils.rs

@@ -0,0 +1,466 @@
+//! Shared frontmatter and metadata utilities for markup extractors.
+//!
+//! This module provides common functionality for extractors that process
+//! documents with YAML frontmatter (Markdown, Djot, etc.).
+//!
+//! This is a core module used by the Djot extractor (always available) and
+//! the enhanced Markdown extractor (requires `office` feature).
+
+use crate::types::Metadata;
+
+use serde_yaml_ng::Value as YamlValue;
+
+/// Extract YAML frontmatter from document content.
+///
+/// Frontmatter is expected to be delimited by `---` or `...` at the start of the document.
+/// This implementation properly handles edge cases:
+/// - `---` appearing within YAML strings or arrays
+/// - Both `---` and `...` as end delimiters (YAML spec compliant)
+/// - Multiline YAML values containing dashes
+///
+/// Returns a tuple of (parsed YAML value, remaining content after frontmatter).
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// let content = "---\ntitle: Test\n---\n\n# Content";
+/// let (yaml, remaining) = extract_frontmatter(content);
+/// assert!(yaml.is_some());
+/// assert!(remaining.contains("# Content"));
+/// ```
+pub fn extract_frontmatter(content: &str) -> (Option<YamlValue>, String) {
+    // Frontmatter must start at the beginning of the document
+    if !content.starts_with("---") {
+        return (None, content.to_string());
+    }
+
+    // Skip opening delimiter
+    let rest = &content[3..];
+
+    // Find the closing delimiter
+    // We need to find "---" or "..." on its own line (not embedded in YAML content)
+    // The delimiter must be preceded by a newline and followed by newline or EOF
+    let mut end_pos = None;
+    let mut search_start = 0;
+
+    while let Some(pos) = rest[search_start..].find('\n') {
+        let absolute_pos = search_start + pos;
+        let after_newline = absolute_pos + 1;
+
+        if after_newline >= rest.len() {
+            break;
+        }
+
+        // Check if we have "---" or "..." at the start of a line
+        let remaining = &rest[after_newline..];
+        if remaining.starts_with("---") || remaining.starts_with("...") {
+            // Verify it's on its own line (followed by newline or EOF)
+            let delimiter_end = after_newline + 3;
+            if delimiter_end >= rest.len() || rest.as_bytes()[delimiter_end] == b'\n' {
+                end_pos = Some(absolute_pos);
+                break;
+            }
+        }
+
+        search_start = after_newline;
+    }
+
+    if let Some(end) = end_pos {
+        let frontmatter_str = &rest[..end];
+        // Skip past the closing delimiter and any following newline
+        let after_delimiter = end + 1; // Skip the newline before delimiter
+        let remaining_start = if after_delimiter + 3 < rest.len() {
+            // Skip "---" or "..."
+            let after_delim = after_delimiter + 3;
+            // Skip trailing newline after delimiter if present
+            if after_delim < rest.len() && rest.as_bytes()[after_delim] == b'\n' {
+                after_delim + 1
+            } else {
+                after_delim
+            }
+        } else {
+            rest.len()
+        };
+
+        let remaining = if remaining_start < rest.len() {
+            &rest[remaining_start..]
+        } else {
+            ""
+        };
+
+        // Try to parse the frontmatter as YAML
+        match serde_yaml_ng::from_str::<YamlValue>(frontmatter_str) {
+            Ok(value) => (Some(value), remaining.to_string()),
+            Err(_) => (None, content.to_string()),
+        }
+    } else {
+        // No closing delimiter found
+        (None, content.to_string())
+    }
+}
+
+/// Extract metadata from YAML frontmatter.
+///
+/// Extracts the following YAML fields into Kreuzberg metadata:
+/// - **Standard fields**: title, author, date, description (as subject)
+/// - **Extended fields**: abstract, subject, category, tags, language, version
+/// - **Array fields** (keywords, tags): converted to comma-separated strings
+///
+/// # Arguments
+///
+/// * `yaml` - The parsed YAML value from frontmatter
+///
+/// # Returns
+///
+/// A `Metadata` struct populated with extracted fields
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// let yaml = serde_yaml_ng::from_str("title: Test\nauthor: John").unwrap();
+/// let metadata = extract_metadata_from_yaml(&yaml);
+/// assert_eq!(metadata.additional.get("title"), Some(&"Test".into()));
+/// ```
+pub fn extract_metadata_from_yaml(yaml: &YamlValue) -> Metadata {
+    let mut metadata = Metadata::default();
+
+    // Title
+    if let Some(title) = yaml.get("title").and_then(|v| v.as_str()) {
+        metadata.additional.insert("title".to_string(), title.into());
+    }
+
+    // Author
+    if let Some(author) = yaml.get("author").and_then(|v| v.as_str()) {
+        metadata.additional.insert("author".to_string(), author.into());
+    }
+
+    // Date (map to created_at)
+    if let Some(date) = yaml.get("date").and_then(|v| v.as_str()) {
+        metadata.created_at = Some(date.to_string());
+    }
+
+    // Keywords (support both string and array)
+    if let Some(keywords) = yaml.get("keywords") {
+        match keywords {
+            YamlValue::String(s) => {
+                metadata.additional.insert("keywords".to_string(), s.clone().into());
+            }
+            YamlValue::Sequence(seq) => {
+                let keywords_str = seq.iter().filter_map(|v| v.as_str()).collect::<Vec<_>>().join(", ");
+                metadata.additional.insert("keywords".to_string(), keywords_str.into());
+            }
+            _ => {}
+        }
+    }
+
+    // Description (map to subject)
+    if let Some(description) = yaml.get("description").and_then(|v| v.as_str()) {
+        metadata.subject = Some(description.to_string());
+    }
+
+    // Abstract
+    if let Some(abstract_text) = yaml.get("abstract").and_then(|v| v.as_str()) {
+        metadata.additional.insert("abstract".to_string(), abstract_text.into());
+    }
+
+    // Subject (overrides description if both present)
+    if let Some(subject) = yaml.get("subject").and_then(|v| v.as_str()) {
+        metadata.subject = Some(subject.to_string());
+    }
+
+    // Category
+    if let Some(category) = yaml.get("category").and_then(|v| v.as_str()) {
+        metadata.additional.insert("category".to_string(), category.into());
+    }
+
+    // Tags (support both string and array)
+    if let Some(tags) = yaml.get("tags") {
+        match tags {
+            YamlValue::String(s) => {
+                metadata.additional.insert("tags".to_string(), s.clone().into());
+            }
+            YamlValue::Sequence(seq) => {
+                let tags_str = seq.iter().filter_map(|v| v.as_str()).collect::<Vec<_>>().join(", ");
+                metadata.additional.insert("tags".to_string(), tags_str.into());
+            }
+            _ => {}
+        }
+    }
+
+    // Language
+    if let Some(language) = yaml.get("language").and_then(|v| v.as_str()) {
+        metadata.additional.insert("language".to_string(), language.into());
+    }
+
+    // Version
+    if let Some(version) = yaml.get("version").and_then(|v| v.as_str()) {
+        metadata.additional.insert("version".to_string(), version.into());
+    }
+
+    metadata
+}
+
+/// Extract first heading as title from content.
+///
+/// Searches for the first level-1 heading (# Title) in the content
+/// and returns it as a potential title if no title was found in frontmatter.
+///
+/// # Arguments
+///
+/// * `content` - The document content to search
+///
+/// # Returns
+///
+/// Some(title) if a heading is found, None otherwise
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// let content = "# My Document\n\nContent here";
+/// assert_eq!(extract_title_from_content(content), Some("My Document".to_string()));
+/// ```
+pub fn extract_title_from_content(content: &str) -> Option<String> {
+    for line in content.lines() {
+        if let Some(heading) = line.strip_prefix("# ") {
+            return Some(heading.trim().to_string());
+        }
+    }
+    None
+}
+
+/// Convert table cells to markdown format.
+///
+/// Takes a 2D array of cell values and formats them as a markdown table
+/// with header row, separator row, and data rows.
+///
+/// # Arguments
+///
+/// * `cells` - A 2D array where cells[0] is the header row
+///
+/// # Returns
+///
+/// A string containing the markdown-formatted table
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// let cells = vec![
+///     vec!["Name".to_string(), "Age".to_string()],
+///     vec!["Alice".to_string(), "30".to_string()],
+/// ];
+/// let markdown = cells_to_markdown(&cells);
+/// assert!(markdown.contains("| Name | Age |"));
+/// ```
+pub fn cells_to_markdown(cells: &[Vec<String>]) -> String {
+    if cells.is_empty() {
+        return String::new();
+    }
+
+    let mut md = String::new();
+
+    // Header row
+    md.push('|');
+    for cell in &cells[0] {
+        md.push(' ');
+        md.push_str(cell);
+        md.push_str(" |");
+    }
+    md.push('\n');
+
+    // Separator row
+    md.push('|');
+    for _ in &cells[0] {
+        md.push_str(" --- |");
+    }
+    md.push('\n');
+
+    // Data rows
+    for row in &cells[1..] {
+        md.push('|');
+        for cell in row {
+            md.push(' ');
+            md.push_str(cell);
+            md.push_str(" |");
+        }
+        md.push('\n');
+    }
+
+    md
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_frontmatter_basic() {
+        let content = "---\ntitle: Test\n---\n\n# Content";
+        let (yaml, remaining) = extract_frontmatter(content);
+
+        assert!(yaml.is_some());
+        assert!(remaining.contains("# Content"));
+
+        let metadata = extract_metadata_from_yaml(&yaml.unwrap());
+        assert_eq!(metadata.additional.get("title").and_then(|v| v.as_str()), Some("Test"));
+    }
+
+    #[test]
+    fn test_frontmatter_with_dashes_in_content() {
+        let content = "---\ntitle: Test\ndescription: |\n  This has ---\n  in the middle\n---\n\n# Body";
+        let (yaml, remaining) = extract_frontmatter(content);
+
+        assert!(yaml.is_some());
+        assert!(remaining.contains("# Body"));
+    }
+
+    #[test]
+    fn test_frontmatter_with_dots_terminator() {
+        let content = "---\ntitle: Test\nauthor: John\n...\n\n# Content";
+        let (yaml, remaining) = extract_frontmatter(content);
+
+        assert!(yaml.is_some());
+        assert!(remaining.contains("# Content"));
+
+        let metadata = extract_metadata_from_yaml(&yaml.unwrap());
+        assert_eq!(metadata.additional.get("title").and_then(|v| v.as_str()), Some("Test"));
+    }
+
+    #[test]
+    fn test_frontmatter_with_triple_dash_in_string() {
+        let content = "---\ntitle: \"Before --- After\"\nauthor: John\n---\n\n# Content";
+        let (yaml, remaining) = extract_frontmatter(content);
+
+        assert!(yaml.is_some());
+        assert!(remaining.contains("# Content"));
+
+        let metadata = extract_metadata_from_yaml(&yaml.unwrap());
+        assert_eq!(
+            metadata.additional.get("title").and_then(|v| v.as_str()),
+            Some("Before --- After")
+        );
+    }
+
+    #[test]
+    fn test_frontmatter_multiline_string_with_dashes() {
+        let content = "---\ntitle: Test\ndescription: |\n  Line 1\n  ---\n  Line 2\n---\n\n# Body";
+        let (yaml, remaining) = extract_frontmatter(content);
+
+        assert!(yaml.is_some());
+        assert!(remaining.contains("# Body"));
+
+        let metadata = extract_metadata_from_yaml(&yaml.unwrap());
+        assert_eq!(metadata.additional.get("title").and_then(|v| v.as_str()), Some("Test"));
+    }
+
+    #[test]
+    fn test_no_frontmatter() {
+        let content = "# Title\n\nContent without frontmatter";
+        let (yaml, remaining) = extract_frontmatter(content);
+
+        assert!(yaml.is_none());
+        assert_eq!(remaining, content);
+    }
+
+    #[test]
+    fn test_incomplete_frontmatter() {
+        let content = "---\ntitle: Test\nauthor: John\n\n# Content";
+        let (yaml, remaining) = extract_frontmatter(content);
+
+        // No closing delimiter, should return None
+        assert!(yaml.is_none());
+        assert_eq!(remaining, content);
+    }
+
+    #[test]
+    fn test_extract_title_from_content() {
+        let content = "# My Document\n\nContent here";
+        assert_eq!(extract_title_from_content(content), Some("My Document".to_string()));
+    }
+
+    #[test]
+    fn test_extract_title_from_content_no_heading() {
+        let content = "Content without heading";
+        assert_eq!(extract_title_from_content(content), None);
+    }
+
+    #[test]
+    fn test_extract_title_from_content_level_2() {
+        let content = "## Subheading\n\nContent";
+        assert_eq!(extract_title_from_content(content), None);
+    }
+
+    #[test]
+    fn test_cells_to_markdown() {
+        let cells = vec![
+            vec!["Name".to_string(), "Age".to_string()],
+            vec!["Alice".to_string(), "30".to_string()],
+            vec!["Bob".to_string(), "25".to_string()],
+        ];
+
+        let markdown = cells_to_markdown(&cells);
+        assert!(markdown.contains("| Name | Age |"));
+        assert!(markdown.contains("| Alice | 30 |"));
+        assert!(markdown.contains("| Bob | 25 |"));
+        assert!(markdown.contains("| --- | --- |"));
+    }
+
+    #[test]
+    fn test_cells_to_markdown_empty() {
+        let cells: Vec<Vec<String>> = vec![];
+        let markdown = cells_to_markdown(&cells);
+        assert_eq!(markdown, "");
+    }
+
+    #[test]
+    fn test_metadata_from_yaml_all_fields() {
+        let yaml_str = r#"
+title: Test Document
+author: John Doe
+date: 2024-01-15
+keywords:
+  - rust
+  - testing
+description: A test document
+abstract: This is an abstract
+subject: Test Subject
+category: Documentation
+tags:
+  - tag1
+  - tag2
+language: en
+version: 1.0
+"#;
+
+        let yaml: YamlValue = serde_yaml_ng::from_str(yaml_str).unwrap();
+        let metadata = extract_metadata_from_yaml(&yaml);
+
+        assert_eq!(
+            metadata.additional.get("title").and_then(|v| v.as_str()),
+            Some("Test Document")
+        );
+        assert_eq!(
+            metadata.additional.get("author").and_then(|v| v.as_str()),
+            Some("John Doe")
+        );
+        assert_eq!(metadata.created_at, Some("2024-01-15".to_string()));
+        assert!(metadata.additional.contains_key("keywords"));
+        assert_eq!(metadata.subject, Some("Test Subject".to_string()));
+        assert!(metadata.additional.contains_key("tags"));
+    }
+
+    #[test]
+    fn test_metadata_from_yaml_string_arrays() {
+        let yaml_str = r#"
+keywords: "single, keyword, string"
+tags: "tag1, tag2"
+"#;
+
+        let yaml: YamlValue = serde_yaml_ng::from_str(yaml_str).unwrap();
+        let metadata = extract_metadata_from_yaml(&yaml);
+
+        assert_eq!(
+            metadata.additional.get("keywords").and_then(|v| v.as_str()),
+            Some("single, keyword, string")
+        );
+    }
+}

data/vendor/kreuzberg/src/extractors/html.rs

@@ -1,7 +1,7 @@
 //! HTML document extractor.
 
 use crate::Result;
-use crate::core::config::ExtractionConfig;
+use crate::core::config::{ExtractionConfig, OutputFormat};
 use crate::extractors::SyncExtractor;
 use crate::plugins::{DocumentExtractor, Plugin};
 use crate::text::utf8_validation;
@@ -202,16 +202,24 @@ impl SyncExtractor for HtmlExtractor {
             .map(|s| s.to_string())
             .unwrap_or_else(|_| String::from_utf8_lossy(content).to_string());
 
-        let (markdown, html_metadata) =
-            crate::extraction::html::convert_html_to_markdown_with_metadata(&html, config.html_options.clone())?;
+        let (content_text, html_metadata) = crate::extraction::html::convert_html_to_markdown_with_metadata(
+            &html,
+            config.html_options.clone(),
+            Some(config.output_format),
+        )?;
 
-        let tables = extract_html_tables(&markdown)?;
+        let tables = extract_html_tables(&content_text)?;
 
-        let content_without_frontmatter = markdown;
+        // Set mime_type based on actual output format
+        let result_mime_type = match config.output_format {
+            OutputFormat::Markdown => "text/markdown",
+            OutputFormat::Djot => "text/djot",
+            _ => mime_type, // Preserve original mime_type for other formats
+        };
 
         Ok(ExtractionResult {
-            content: content_without_frontmatter,
-            mime_type: mime_type.to_string(),
+            content: content_text,
+            mime_type: result_mime_type.to_string(),
             metadata: Metadata {
                 format: html_metadata.map(|m| crate::types::FormatMetadata::Html(Box::new(m))),
                 ..Default::default()
@@ -221,6 +229,8 @@ impl SyncExtractor for HtmlExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
+            elements: None,
         })
     }
 }
@@ -275,7 +285,7 @@ mod tests {
 
     /// Helper function to convert HTML to markdown for testing
     fn html_to_markdown_for_test(html: &str) -> String {
-        crate::extraction::html::convert_html_to_markdown(html, None).unwrap()
+        crate::extraction::html::convert_html_to_markdown(html, None, None).unwrap()
     }
 
     #[test]
@@ -416,4 +426,66 @@ mod tests {
         assert_eq!(table.cells[1], vec!["Alice", "30"]);
         assert_eq!(table.cells[2], vec!["Bob", "25"]);
     }
+
+    #[tokio::test]
+    async fn test_html_extractor_with_djot_output() {
+        let html = r#"
+        <html>
+            <body>
+                <h1>Test Page</h1>
+                <p>Content with <strong>emphasis</strong>.</p>
+            </body>
+        </html>
+    "#;
+
+        let extractor = HtmlExtractor::new();
+        let config = ExtractionConfig {
+            output_format: OutputFormat::Djot,
+            ..Default::default()
+        };
+
+        let result = extractor
+            .extract_bytes(html.as_bytes(), "text/html", &config)
+            .await
+            .unwrap();
+
+        assert_eq!(result.mime_type, "text/djot");
+        assert!(result.content.contains("# Test Page"));
+        assert!(result.content.contains("*emphasis*")); // Djot strong syntax
+    }
+
+    #[tokio::test]
+    async fn test_html_extractor_djot_double_conversion_prevention() {
+        let html = r#"
+        <html>
+            <body>
+                <h1>Test</h1>
+                <p>Content with <strong>bold</strong> text.</p>
+            </body>
+        </html>
+    "#;
+
+        let extractor = HtmlExtractor::new();
+        let config = ExtractionConfig {
+            output_format: OutputFormat::Djot,
+            ..Default::default()
+        };
+
+        let result = extractor
+            .extract_bytes(html.as_bytes(), "text/html", &config)
+            .await
+            .unwrap();
+
+        // Content should already be in djot format
+        assert_eq!(result.mime_type, "text/djot");
+        let original_content = result.content.clone();
+
+        // Simulate pipeline format application
+        let mut pipeline_result = result.clone();
+        crate::core::pipeline::apply_output_format(&mut pipeline_result, OutputFormat::Djot);
+
+        // Content should be identical - no re-conversion should occur
+        assert_eq!(pipeline_result.content, original_content);
+        assert_eq!(pipeline_result.mime_type, "text/djot");
+    }
 }