kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/extractors/markdown.rs

@@ -12,6 +12,10 @@
 //!
 //! Requires the `office` feature (which includes `pulldown-cmark`).
 
+#[cfg(feature = "office")]
+use super::frontmatter_utils::{
+    cells_to_markdown, extract_frontmatter, extract_metadata_from_yaml, extract_title_from_content,
+};
 #[cfg(feature = "office")]
 use crate::Result;
 #[cfg(feature = "office")]
@@ -24,8 +28,6 @@ use crate::types::{ExtractionResult, Metadata, Table};
 use async_trait::async_trait;
 #[cfg(feature = "office")]
 use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
-#[cfg(feature = "office")]
-use serde_yaml_ng::Value as YamlValue;
 
 /// Enhanced Markdown extractor with metadata and table support.
 ///
@@ -44,102 +46,7 @@ impl MarkdownExtractor {
         Self
     }
 
-    /// Extract YAML frontmatter from markdown content.
-    ///
-    /// Frontmatter is expected to be delimited by `---` at the start of the document.
-    /// Returns the remaining content after frontmatter.
-    fn extract_frontmatter(content: &str) -> (Option<YamlValue>, String) {
-        if !content.starts_with("---") {
-            return (None, content.to_string());
-        }
-
-        let rest = &content[3..];
-        if let Some(end_pos) = rest.find("\n---") {
-            let frontmatter_str = &rest[..end_pos];
-            let remaining = &rest[end_pos + 4..];
-
-            match serde_yaml_ng::from_str::<YamlValue>(frontmatter_str) {
-                Ok(value) => (Some(value), remaining.to_string()),
-                Err(_) => (None, content.to_string()),
-            }
-        } else {
-            (None, content.to_string())
-        }
-    }
-
-    /// Extract metadata from YAML frontmatter.
-    ///
-    /// Extracts the following YAML fields:
-    /// - Standard fields: title, author, date, description (as subject)
-    /// - Extended fields: abstract, subject, category, tags, language, version
-    /// - Array fields (keywords, tags): converted to comma-separated strings
-    fn extract_metadata_from_yaml(yaml: &YamlValue) -> Metadata {
-        let mut metadata = Metadata::default();
-
-        if let Some(title) = yaml.get("title").and_then(|v| v.as_str()) {
-            metadata.additional.insert("title".to_string(), title.into());
-        }
-
-        if let Some(author) = yaml.get("author").and_then(|v| v.as_str()) {
-            metadata.additional.insert("author".to_string(), author.into());
-        }
-
-        if let Some(date) = yaml.get("date").and_then(|v| v.as_str()) {
-            metadata.created_at = Some(date.to_string());
-        }
-
-        if let Some(keywords) = yaml.get("keywords") {
-            match keywords {
-                YamlValue::String(s) => {
-                    metadata.additional.insert("keywords".to_string(), s.clone().into());
-                }
-                YamlValue::Sequence(seq) => {
-                    let keywords_str = seq.iter().filter_map(|v| v.as_str()).collect::<Vec<_>>().join(", ");
-                    metadata.additional.insert("keywords".to_string(), keywords_str.into());
-                }
-                _ => {}
-            }
-        }
-
-        if let Some(description) = yaml.get("description").and_then(|v| v.as_str()) {
-            metadata.subject = Some(description.to_string());
-        }
-
-        if let Some(abstract_text) = yaml.get("abstract").and_then(|v| v.as_str()) {
-            metadata.additional.insert("abstract".to_string(), abstract_text.into());
-        }
-
-        if let Some(subject) = yaml.get("subject").and_then(|v| v.as_str()) {
-            metadata.subject = Some(subject.to_string());
-        }
-
-        if let Some(category) = yaml.get("category").and_then(|v| v.as_str()) {
-            metadata.additional.insert("category".to_string(), category.into());
-        }
-
-        if let Some(tags) = yaml.get("tags") {
-            match tags {
-                YamlValue::String(s) => {
-                    metadata.additional.insert("tags".to_string(), s.clone().into());
-                }
-                YamlValue::Sequence(seq) => {
-                    let tags_str = seq.iter().filter_map(|v| v.as_str()).collect::<Vec<_>>().join(", ");
-                    metadata.additional.insert("tags".to_string(), tags_str.into());
-                }
-                _ => {}
-            }
-        }
-
-        if let Some(language) = yaml.get("language").and_then(|v| v.as_str()) {
-            metadata.additional.insert("language".to_string(), language.into());
-        }
-
-        if let Some(version) = yaml.get("version").and_then(|v| v.as_str()) {
-            metadata.additional.insert("version".to_string(), version.into());
-        }
-
-        metadata
-    }
+    // Frontmatter utilities moved to shared frontmatter_utils module
 
     /// Extract plain text from markdown AST.
     fn extract_text_from_events(events: &[Event]) -> String {
@@ -222,7 +129,7 @@ impl MarkdownExtractor {
                     if let Some((cells, idx)) = current_table.take()
                         && !cells.is_empty()
                     {
-                        let markdown = Self::cells_to_markdown(&cells);
+                        let markdown = cells_to_markdown(&cells);
                         tables.push(Table {
                             cells,
                             markdown,
@@ -238,50 +145,7 @@ impl MarkdownExtractor {
         tables
     }
 
-    /// Convert table cells to markdown format.
-    fn cells_to_markdown(cells: &[Vec<String>]) -> String {
-        if cells.is_empty() {
-            return String::new();
-        }
-
-        let mut md = String::new();
-
-        md.push('|');
-        for cell in &cells[0] {
-            md.push(' ');
-            md.push_str(cell);
-            md.push_str(" |");
-        }
-        md.push('\n');
-
-        md.push('|');
-        for _ in &cells[0] {
-            md.push_str(" --- |");
-        }
-        md.push('\n');
-
-        for row in &cells[1..] {
-            md.push('|');
-            for cell in row {
-                md.push(' ');
-                md.push_str(cell);
-                md.push_str(" |");
-            }
-            md.push('\n');
-        }
-
-        md
-    }
-
-    /// Extract first heading as title if not in frontmatter.
-    fn extract_title_from_content(content: &str) -> Option<String> {
-        for line in content.lines() {
-            if let Some(heading) = line.strip_prefix("# ") {
-                return Some(heading.trim().to_string());
-            }
-        }
-        None
-    }
+    // cells_to_markdown and extract_title_from_content moved to shared frontmatter_utils module
 }
 
 #[cfg(feature = "office")]
@@ -336,16 +200,16 @@ impl DocumentExtractor for MarkdownExtractor {
     ) -> Result<ExtractionResult> {
         let text = String::from_utf8_lossy(content).into_owned();
 
-        let (yaml, remaining_content) = Self::extract_frontmatter(&text);
+        let (yaml, remaining_content) = extract_frontmatter(&text);
 
         let mut metadata = if let Some(ref yaml_value) = yaml {
-            Self::extract_metadata_from_yaml(yaml_value)
+            extract_metadata_from_yaml(yaml_value)
         } else {
             Metadata::default()
         };
 
         if !metadata.additional.contains_key("title")
-            && let Some(title) = Self::extract_title_from_content(&remaining_content)
+            && let Some(title) = extract_title_from_content(&remaining_content)
         {
             metadata.additional.insert("title".to_string(), title.into());
         }
@@ -365,7 +229,9 @@ impl DocumentExtractor for MarkdownExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         })
     }
 
@@ -380,7 +246,9 @@ impl DocumentExtractor for MarkdownExtractor {
 
 #[cfg(all(test, feature = "office"))]
 mod tests {
+    use super::super::frontmatter_utils::{cells_to_markdown, extract_frontmatter, extract_metadata_from_yaml};
     use super::*;
+    use serde_yaml_ng::Value as YamlValue;
 
     #[test]
     fn test_can_extract_markdown_mime_types() {
@@ -399,7 +267,7 @@ mod tests {
             b"# Header\n\nThis is a paragraph with **bold** and *italic* text.\n\n## Subheading\n\nMore content here.";
         let text = String::from_utf8_lossy(content).into_owned();
 
-        let (yaml, remaining) = MarkdownExtractor::extract_frontmatter(&text);
+        let (yaml, remaining) = extract_frontmatter(&text);
         assert!(yaml.is_none());
         assert!(!remaining.is_empty());
 
@@ -419,19 +287,25 @@ mod tests {
 
         let text = String::from_utf8_lossy(content).into_owned();
 
-        let (yaml_opt, remaining) = MarkdownExtractor::extract_frontmatter(&text);
+        let (yaml_opt, remaining) = extract_frontmatter(&text);
         assert!(yaml_opt.is_some());
         assert!(remaining.contains("# Content"));
 
         let yaml = yaml_opt.expect("Should extract YAML frontmatter");
-        let metadata = MarkdownExtractor::extract_metadata_from_yaml(&yaml);
+        let metadata = extract_metadata_from_yaml(&yaml);
 
         assert_eq!(
-            metadata.additional.get("title").and_then(|v| v.as_str()),
+            metadata
+                .additional
+                .get("title")
+                .and_then(|v: &serde_json::Value| v.as_str()),
             Some("My Document")
         );
         assert_eq!(
-            metadata.additional.get("author").and_then(|v| v.as_str()),
+            metadata
+                .additional
+                .get("author")
+                .and_then(|v: &serde_json::Value| v.as_str()),
             Some("John Doe")
         );
         assert_eq!(metadata.created_at, Some("2024-01-15".to_string()));
@@ -450,13 +324,16 @@ mod tests {
         let content = b"---\ntitle: Document\nkeywords:\n  - rust\n  - markdown\n  - parsing\n---\n\nContent";
 
         let text = String::from_utf8_lossy(content).into_owned();
-        let (yaml_opt, _remaining) = MarkdownExtractor::extract_frontmatter(&text);
+        let (yaml_opt, _remaining) = extract_frontmatter(&text);
 
         assert!(yaml_opt.is_some());
         let yaml = yaml_opt.expect("Should extract YAML frontmatter");
-        let metadata = MarkdownExtractor::extract_metadata_from_yaml(&yaml);
+        let metadata = extract_metadata_from_yaml(&yaml);
 
-        let keywords = metadata.additional.get("keywords").and_then(|v| v.as_str());
+        let keywords = metadata
+            .additional
+            .get("keywords")
+            .and_then(|v: &serde_json::Value| v.as_str());
         assert!(keywords.is_some());
         let keywords_str = keywords.expect("Should extract keywords from metadata");
         assert!(keywords_str.contains("rust"));
@@ -485,11 +362,11 @@ mod tests {
         let content = b"# Main Title\n\nSome content\n\nMore text";
         let text = String::from_utf8_lossy(content).into_owned();
 
-        let (yaml, remaining) = MarkdownExtractor::extract_frontmatter(&text);
+        let (yaml, remaining) = extract_frontmatter(&text);
         assert!(yaml.is_none());
         assert_eq!(remaining, text);
 
-        let title = MarkdownExtractor::extract_title_from_content(&remaining);
+        let title = extract_title_from_content(&remaining);
         assert_eq!(title, Some("Main Title".to_string()));
     }
 
@@ -498,7 +375,7 @@ mod tests {
         let content = b"";
         let text = String::from_utf8_lossy(content).into_owned();
 
-        let (yaml, remaining) = MarkdownExtractor::extract_frontmatter(&text);
+        let (yaml, remaining) = extract_frontmatter(&text);
         assert!(yaml.is_none());
         assert!(remaining.is_empty());
 
@@ -513,7 +390,7 @@ mod tests {
         let content = b"   \n\n  \n";
         let text = String::from_utf8_lossy(content).into_owned();
 
-        let (yaml, remaining) = MarkdownExtractor::extract_frontmatter(&text);
+        let (yaml, remaining) = extract_frontmatter(&text);
         assert!(yaml.is_none());
 
         let parser = Parser::new_ext(&remaining, Options::ENABLE_TABLES);
@@ -528,7 +405,7 @@ mod tests {
 
         let text = String::from_utf8_lossy(content).into_owned();
 
-        let (yaml, remaining) = MarkdownExtractor::extract_frontmatter(&text);
+        let (yaml, remaining) = extract_frontmatter(&text);
         assert!(yaml.is_none());
 
         let parser = Parser::new_ext(&remaining, Options::ENABLE_TABLES);
@@ -580,7 +457,7 @@ mod tests {
             vec!["Data 3".to_string(), "Data 4".to_string()],
         ];
 
-        let markdown = MarkdownExtractor::cells_to_markdown(&cells);
+        let markdown = cells_to_markdown(&cells);
         assert!(markdown.contains("Header 1"));
         assert!(markdown.contains("Data 1"));
         assert!(markdown.contains("---"));
@@ -619,7 +496,7 @@ mod tests {
         let content = b"---\nthis: is: invalid: yaml:\n---\n\nContent here";
         let text = String::from_utf8_lossy(content).into_owned();
 
-        let (yaml, _remaining) = MarkdownExtractor::extract_frontmatter(&text);
+        let (yaml, _remaining) = extract_frontmatter(&text);
         let _ = yaml;
     }
 
@@ -650,7 +527,7 @@ nested:
 "#;
 
         let yaml: YamlValue = serde_yaml_ng::from_str(yaml_str).expect("Valid YAML");
-        let metadata = MarkdownExtractor::extract_metadata_from_yaml(&yaml);
+        let metadata = extract_metadata_from_yaml(&yaml);
 
         assert_eq!(metadata.created_at, Some("2024-01-15".to_string()));
         assert_eq!(

data/vendor/kreuzberg/src/extractors/mod.rs

@@ -64,6 +64,9 @@ pub trait SyncExtractor {
 pub mod structured;
 pub mod text;
 
+pub mod djot_format;
+pub mod frontmatter_utils;
+
 #[cfg(feature = "archives")]
 pub mod security;
 
@@ -166,6 +169,8 @@ pub use epub::EpubExtractor;
 #[cfg(feature = "office")]
 pub use fictionbook::FictionBookExtractor;
 
+pub use djot_format::DjotExtractor;
+
 #[cfg(feature = "office")]
 pub use markdown::MarkdownExtractor as EnhancedMarkdownExtractor;
 
@@ -281,6 +286,8 @@ pub fn register_default_extractors() -> Result<()> {
     #[cfg(feature = "excel")]
     registry.register(Arc::new(ExcelExtractor::new()))?;
 
+    registry.register(Arc::new(DjotExtractor::new()))?;
+
     #[cfg(feature = "office")]
     {
         registry.register(Arc::new(EnhancedMarkdownExtractor::new()))?;
@@ -341,10 +348,11 @@ mod tests {
         let extractor_names = reg.list();
 
         #[allow(unused_mut)]
-        let mut expected_count = 3;
+        let mut expected_count = 4; // plain-text, markdown, structured, djot
         assert!(extractor_names.contains(&"plain-text-extractor".to_string()));
         assert!(extractor_names.contains(&"markdown-extractor".to_string()));
         assert!(extractor_names.contains(&"structured-extractor".to_string()));
+        assert!(extractor_names.contains(&"djot-extractor".to_string()));
 
         #[cfg(feature = "ocr")]
         {

data/vendor/kreuzberg/src/extractors/odt.rs

@@ -567,6 +567,8 @@ impl DocumentExtractor for OdtExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
+            elements: None,
         })
     }
 

data/vendor/kreuzberg/src/extractors/opml/core.rs

@@ -0,0 +1,165 @@
+//! Core OPML extractor implementation.
+//!
+//! This module provides the main `OpmlExtractor` struct and implements the
+//! `Plugin` and `DocumentExtractor` traits for OPML document processing.
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::{ExtractionResult, Metadata};
+use async_trait::async_trait;
+
+#[cfg(feature = "office")]
+use super::parser;
+
+/// OPML format extractor.
+///
+/// Extracts outline structure and metadata from OPML documents using native Rust parsing.
+pub struct OpmlExtractor;
+
+impl OpmlExtractor {
+    /// Create a new OPML extractor.
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for OpmlExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Plugin for OpmlExtractor {
+    fn name(&self) -> &str {
+        "opml-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn description(&self) -> &str {
+        "Extracts content and metadata from OPML (Outline Processor Markup Language) documents"
+    }
+
+    fn author(&self) -> &str {
+        "Kreuzberg Team"
+    }
+}
+
+#[cfg(feature = "office")]
+#[async_trait]
+impl DocumentExtractor for OpmlExtractor {
+    #[cfg_attr(
+        feature = "otel",
+        tracing::instrument(
+            skip(self, content, _config),
+            fields(
+                extractor.name = self.name(),
+                content.size_bytes = content.len(),
+            )
+        )
+    )]
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let (extracted_content, metadata_map) = parser::extract_content_and_metadata(content)?;
+
+        Ok(ExtractionResult {
+            content: extracted_content,
+            mime_type: mime_type.to_string(),
+            metadata: Metadata {
+                additional: metadata_map,
+                ..Default::default()
+            },
+            pages: None,
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            djot_content: None,
+            elements: None,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &["text/x-opml", "application/xml+opml"]
+    }
+
+    fn priority(&self) -> i32 {
+        55
+    }
+}
+
+#[cfg(all(test, feature = "office"))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_opml_extractor_plugin_interface() {
+        let extractor = OpmlExtractor::new();
+        assert_eq!(extractor.name(), "opml-extractor");
+        assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
+        assert_eq!(extractor.priority(), 55);
+        assert!(!extractor.supported_mime_types().is_empty());
+    }
+
+    #[test]
+    fn test_opml_extractor_default() {
+        let extractor = OpmlExtractor;
+        assert_eq!(extractor.name(), "opml-extractor");
+    }
+
+    #[tokio::test]
+    async fn test_opml_extractor_initialize_shutdown() {
+        let extractor = OpmlExtractor::new();
+        assert!(extractor.initialize().is_ok());
+        assert!(extractor.shutdown().is_ok());
+    }
+
+    #[test]
+    fn test_opml_supported_mime_types() {
+        let extractor = OpmlExtractor::new();
+        let supported = extractor.supported_mime_types();
+        assert!(supported.contains(&"text/x-opml"));
+        assert!(supported.contains(&"application/xml+opml"));
+    }
+
+    #[tokio::test]
+    async fn test_opml_extractor_async_extraction() {
+        let extractor = OpmlExtractor::new();
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Async Test</title>
+  </head>
+  <body>
+    <outline text="Item" />
+  </body>
+</opml>"#;
+
+        let result = extractor
+            .extract_bytes(opml, "text/x-opml", &ExtractionConfig::default())
+            .await
+            .expect("Should extract OPML asynchronously");
+
+        assert_eq!(result.mime_type, "text/x-opml");
+        assert!(result.content.contains("Item"));
+        assert_eq!(
+            result.metadata.additional.get("title").and_then(|v| v.as_str()),
+            Some("Async Test")
+        );
+    }
+}

data/vendor/kreuzberg/src/extractors/opml/mod.rs

@@ -0,0 +1,31 @@
+//! Native OPML (Outline Processor Markup Language) extractor using the `roxmltree` library.
+//!
+//! This extractor provides native Rust-based OPML extraction, parsing outline structures
+//! commonly used for RSS feed lists, podcast directories, and general outlines.
+//!
+//! Extracts:
+//! - Metadata from `<head>`: title, dateCreated, dateModified, ownerName, ownerEmail
+//! - Content from `<body><outline>` hierarchy using text attributes
+//! - Outline hierarchy structure preserved in plain text format with indentation
+//! - Note: URLs (xmlUrl, htmlUrl) are extracted from attributes but not included in main content
+//!
+//! Example OPML structure:
+//! ```xml
+//! <opml version="2.0">
+//!   <head>
+//!     <title>My Feeds</title>
+//!     <ownerName>John</ownerName>
+//!   </head>
+//!   <body>
+//!     <outline text="Tech" type="folder">
+//!       <outline text="Hacker News" type="rss" xmlUrl="https://..." />
+//!     </outline>
+//!   </body>
+//! </opml>
+//! ```
+
+mod core;
+mod parser;
+
+// Re-export public API
+pub use core::OpmlExtractor;