kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/extraction/transform/types.rs

@@ -0,0 +1,27 @@
+//! Type definitions for transformation operations.
+
+/// Metadata about a detected list item.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ListItemMetadata {
+    /// Type of list (Bullet, Numbered, etc.)
+    pub list_type: ListType,
+    /// Starting byte offset in the content string
+    pub byte_start: usize,
+    /// Ending byte offset in the content string
+    pub byte_end: usize,
+    /// List item indent level
+    pub indent_level: u32,
+}
+
+/// Type of list detection.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ListType {
+    /// Bullet points (-, *, •, etc.)
+    Bullet,
+    /// Numbered lists (1., 2., etc.)
+    Numbered,
+    /// Lettered lists (a., b., A., B., etc.)
+    Lettered,
+    /// Indented items
+    Indented,
+}

data/vendor/kreuzberg/src/extractors/archive.rs

@@ -78,6 +78,8 @@ fn build_archive_result(
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     }
 }
 

data/vendor/kreuzberg/src/extractors/bibtex.rs

@@ -172,6 +172,8 @@ impl DocumentExtractor for BibtexExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
+            elements: None,
         })
     }
 

data/vendor/kreuzberg/src/extractors/djot_format/attributes.rs

@@ -0,0 +1,134 @@
+//! Djot attribute parsing utilities.
+//!
+//! Handles parsing of Djot attributes from jotdown events and string syntax.
+
+use std::collections::HashMap;
+
+/// Parse jotdown attributes into our Attributes representation.
+///
+/// Converts jotdown's internal attribute representation to Kreuzberg's
+/// standardized Attributes struct, handling IDs, classes, and key-value pairs.
+pub fn parse_jotdown_attributes(attrs: &jotdown::Attributes) -> crate::types::Attributes {
+    use crate::types::Attributes;
+    use jotdown::AttributeKind;
+
+    let mut id = None;
+    let mut classes = Vec::new();
+    let mut key_values = HashMap::new();
+
+    for (kind, value) in attrs.iter() {
+        match kind {
+            AttributeKind::Id => {
+                // Last ID wins if multiple are specified
+                id = Some(value.to_string());
+            }
+            AttributeKind::Class => {
+                classes.push(value.to_string());
+            }
+            AttributeKind::Pair { key } => {
+                key_values.insert(key.to_string(), value.to_string());
+            }
+            AttributeKind::Comment => {
+                // Comments are ignored in our representation
+            }
+        }
+    }
+
+    Attributes {
+        id,
+        classes,
+        key_values,
+    }
+}
+
+/// Parse djot attribute syntax from string: {.class #id key="value"}
+#[allow(dead_code)]
+pub fn parse_djot_attributes(attr_str: &str) -> crate::types::Attributes {
+    use crate::types::Attributes;
+
+    let mut attrs = Attributes {
+        id: None,
+        classes: Vec::new(),
+        key_values: HashMap::new(),
+    };
+
+    // Simple parser for attribute syntax
+    let tokens = attr_str.split_whitespace();
+
+    for token in tokens {
+        if let Some(class) = token.strip_prefix('.') {
+            // Class
+            attrs.classes.push(class.to_string());
+        } else if let Some(id) = token.strip_prefix('#') {
+            // ID
+            attrs.id = Some(id.to_string());
+        } else if token.contains('=') {
+            // Key-value pair
+            if let Some((key, value)) = token.split_once('=') {
+                let clean_value = value.trim_matches('"').trim_matches('\'');
+                attrs.key_values.insert(key.to_string(), clean_value.to_string());
+            }
+        }
+    }
+
+    attrs
+}
+
+/// Render attributes to djot attribute syntax.
+///
+/// Converts Kreuzberg's Attributes struct back to djot attribute syntax:
+/// {.class #id key="value"}
+pub fn render_attributes(attrs: &crate::types::Attributes) -> String {
+    let mut parts = Vec::new();
+
+    if let Some(ref id) = attrs.id {
+        parts.push(format!("#{}", id));
+    }
+
+    for class in &attrs.classes {
+        parts.push(format!(".{}", class));
+    }
+
+    for (key, value) in &attrs.key_values {
+        parts.push(format!("{}=\"{}\"", key, value));
+    }
+
+    if parts.is_empty() {
+        String::new()
+    } else {
+        format!("{{{}}}", parts.join(" "))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_render_attributes_with_all_parts() {
+        let mut attrs = crate::types::Attributes {
+            id: Some("my-id".to_string()),
+            classes: vec!["class1".to_string(), "class2".to_string()],
+            key_values: HashMap::new(),
+        };
+        attrs.key_values.insert("data-test".to_string(), "value".to_string());
+
+        let rendered = render_attributes(&attrs);
+        assert!(rendered.contains("#my-id"));
+        assert!(rendered.contains(".class1"));
+        assert!(rendered.contains(".class2"));
+        assert!(rendered.contains("data-test"));
+    }
+
+    #[test]
+    fn test_render_attributes_empty() {
+        let attrs = crate::types::Attributes {
+            id: None,
+            classes: vec![],
+            key_values: HashMap::new(),
+        };
+
+        let rendered = render_attributes(&attrs);
+        assert_eq!(rendered, "");
+    }
+}

data/vendor/kreuzberg/src/extractors/djot_format/conversion.rs

@@ -0,0 +1,223 @@
+//! Djot content conversion and HTML rendering APIs.
+//!
+//! Provides public APIs for converting between different representations:
+//! - DjotContent to djot markup
+//! - ExtractionResult to djot markup
+//! - Djot markup to HTML
+
+use super::rendering::render_block_to_djot;
+use jotdown::Parser;
+
+/// Convert DjotContent back to djot markup.
+///
+/// This function takes a `DjotContent` structure and generates valid djot markup
+/// from it, preserving:
+/// - Block structure (headings, code blocks, lists, blockquotes, etc.)
+/// - Inline formatting (strong, emphasis, highlight, subscript, superscript, etc.)
+/// - Attributes where present ({.class #id key="value"})
+///
+/// # Arguments
+///
+/// * `content` - The DjotContent to convert
+///
+/// # Returns
+///
+/// A String containing valid djot markup
+///
+/// # Example
+///
+/// ```ignore
+/// let djot_content = // ... extract from some source
+/// let markup = djot_content_to_djot(&djot_content);
+/// println!("{}", markup);
+/// ```
+pub fn djot_content_to_djot(content: &crate::types::DjotContent) -> String {
+    let mut output = String::new();
+
+    for block in &content.blocks {
+        render_block_to_djot(&mut output, block, 0);
+    }
+
+    output
+}
+
+/// Convert any ExtractionResult to djot format.
+///
+/// This function converts an `ExtractionResult` to djot markup:
+/// - If `djot_content` is `Some`, uses `djot_content_to_djot` for full fidelity conversion
+/// - Otherwise, wraps the plain text content in paragraphs
+///
+/// # Arguments
+///
+/// * `result` - The ExtractionResult to convert
+///
+/// # Returns
+///
+/// A `Result` containing the djot markup string
+///
+/// # Example
+///
+/// ```ignore
+/// let result = extractor.extract_bytes(bytes, "text/plain", &config).await?;
+/// let djot_markup = extraction_result_to_djot(&result)?;
+/// ```
+pub fn extraction_result_to_djot(result: &crate::types::ExtractionResult) -> crate::Result<String> {
+    if let Some(ref djot_content) = result.djot_content {
+        Ok(djot_content_to_djot(djot_content))
+    } else {
+        // Convert plain text to basic djot paragraphs
+        let mut output = String::new();
+
+        // Split content by double newlines to create paragraphs
+        let paragraphs: Vec<&str> = result.content.split("\n\n").collect();
+
+        for para in paragraphs {
+            let trimmed = para.trim();
+            if !trimmed.is_empty() {
+                output.push_str(trimmed);
+                output.push_str("\n\n");
+            }
+        }
+
+        Ok(output)
+    }
+}
+
+/// Render djot content to HTML.
+///
+/// This function takes djot source text and renders it to HTML using jotdown's
+/// built-in HTML renderer.
+///
+/// # Arguments
+///
+/// * `djot_source` - The djot markup text to render
+///
+/// # Returns
+///
+/// A `Result` containing the rendered HTML string
+///
+/// # Example
+///
+/// ```ignore
+/// let djot = "# Hello\n\nThis is *bold* and _italic_.";
+/// let html = djot_to_html(djot)?;
+/// assert!(html.contains("<h1>"));
+/// assert!(html.contains("<strong>"));
+/// assert!(html.contains("<em>"));
+/// ```
+pub fn djot_to_html(djot_source: &str) -> crate::Result<String> {
+    let parser = Parser::new(djot_source);
+    let html = jotdown::html::render_to_string(parser);
+    Ok(html)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::types::{BlockType, DjotContent, ExtractionResult, FormattedBlock, InlineElement, InlineType, Metadata};
+
+    #[test]
+    fn test_djot_content_to_djot_heading() {
+        let content = DjotContent {
+            plain_text: "Test Heading".to_string(),
+            blocks: vec![FormattedBlock {
+                block_type: BlockType::Heading,
+                level: Some(1),
+                inline_content: vec![InlineElement {
+                    element_type: InlineType::Text,
+                    content: "Test Heading".to_string(),
+                    attributes: None,
+                    metadata: None,
+                }],
+                attributes: None,
+                language: None,
+                code: None,
+                children: vec![],
+            }],
+            metadata: Metadata::default(),
+            tables: vec![],
+            images: vec![],
+            links: vec![],
+            footnotes: vec![],
+            attributes: Default::default(),
+        };
+
+        let markup = djot_content_to_djot(&content);
+        assert!(markup.contains("# Test Heading"));
+    }
+
+    #[test]
+    fn test_extraction_result_to_djot_with_djot_content() {
+        let result = ExtractionResult {
+            content: "Test content".to_string(),
+            mime_type: "text/djot".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            djot_content: Some(DjotContent {
+                plain_text: "Test content".to_string(),
+                blocks: vec![FormattedBlock {
+                    block_type: BlockType::Paragraph,
+                    level: None,
+                    inline_content: vec![InlineElement {
+                        element_type: InlineType::Text,
+                        content: "Test content".to_string(),
+                        attributes: None,
+                        metadata: None,
+                    }],
+                    attributes: None,
+                    language: None,
+                    code: None,
+                    children: vec![],
+                }],
+                metadata: Metadata::default(),
+                tables: vec![],
+                images: vec![],
+                links: vec![],
+                footnotes: vec![],
+                attributes: Default::default(),
+            }),
+            elements: None,
+        };
+
+        let markup = extraction_result_to_djot(&result).expect("Should convert");
+        assert!(markup.contains("Test content"));
+    }
+
+    #[test]
+    fn test_extraction_result_to_djot_without_djot_content() {
+        let result = ExtractionResult {
+            content: "Paragraph one\n\nParagraph two".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            djot_content: None,
+            elements: None,
+        };
+
+        let markup = extraction_result_to_djot(&result).expect("Should convert");
+        assert!(markup.contains("Paragraph one"));
+        assert!(markup.contains("Paragraph two"));
+    }
+
+    #[test]
+    fn test_djot_to_html_heading() {
+        let djot = "# Hello";
+        let html = djot_to_html(djot).expect("Should render");
+        assert!(html.contains("<h1>") || html.contains("<H1>"));
+    }
+
+    #[test]
+    fn test_djot_to_html_formatting() {
+        let djot = "This is *bold* and _italic_.";
+        let html = djot_to_html(djot).expect("Should render");
+        assert!(html.contains("<strong>") || html.contains("<em>"));
+    }
+}

data/vendor/kreuzberg/src/extractors/djot_format/extractor.rs

@@ -0,0 +1,172 @@
+//! Djot document extractor with plugin integration.
+//!
+//! Implements the DocumentExtractor and Plugin traits for Djot markup files.
+
+use super::parsing::{extract_complete_djot_content, extract_tables_from_events, extract_text_from_events};
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::{ExtractionResult, Metadata};
+use async_trait::async_trait;
+use jotdown::{Event, Parser};
+
+/// Djot markup extractor with metadata and table support.
+///
+/// Parses Djot documents with YAML frontmatter, extracting:
+/// - Metadata from YAML frontmatter
+/// - Plain text content
+/// - Tables as structured data
+/// - Document structure (headings, links, code blocks)
+#[derive(Debug, Clone)]
+pub struct DjotExtractor;
+
+impl DjotExtractor {
+    /// Create a new Djot extractor.
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for DjotExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Plugin for DjotExtractor {
+    fn name(&self) -> &str {
+        "djot-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn description(&self) -> &str {
+        "Extracts content from Djot markup files with YAML frontmatter and table support"
+    }
+
+    fn author(&self) -> &str {
+        "Kreuzberg Team"
+    }
+}
+
+#[async_trait]
+impl DocumentExtractor for DjotExtractor {
+    #[cfg_attr(
+        feature = "otel",
+        tracing::instrument(
+            skip(self, content, _config),
+            fields(
+                extractor.name = self.name(),
+                content.size_bytes = content.len(),
+            )
+        )
+    )]
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let text = String::from_utf8_lossy(content).into_owned();
+
+        let (yaml, remaining_content) = crate::extractors::frontmatter_utils::extract_frontmatter(&text);
+
+        let mut metadata = if let Some(ref yaml_value) = yaml {
+            crate::extractors::frontmatter_utils::extract_metadata_from_yaml(yaml_value)
+        } else {
+            Metadata::default()
+        };
+
+        if !metadata.additional.contains_key("title")
+            && let Some(title) = crate::extractors::frontmatter_utils::extract_title_from_content(&remaining_content)
+        {
+            metadata.additional.insert("title".to_string(), title.into());
+        }
+
+        // Parse with jotdown and collect events once for extraction
+        let parser = Parser::new(&remaining_content);
+        let events: Vec<Event> = parser.collect();
+
+        let extracted_text = extract_text_from_events(&events);
+        let tables = extract_tables_from_events(&events);
+
+        // Extract complete djot content with all features
+        let djot_content = extract_complete_djot_content(&events, metadata.clone(), tables.clone());
+
+        Ok(ExtractionResult {
+            content: extracted_text,
+            mime_type: mime_type.to_string(),
+            metadata,
+            tables,
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            djot_content: Some(djot_content),
+            elements: None,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &["text/djot", "text/x-djot"]
+    }
+
+    fn priority(&self) -> i32 {
+        50
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_djot_extractor_creation() {
+        let extractor = DjotExtractor::new();
+        assert_eq!(extractor.name(), "djot-extractor");
+    }
+
+    #[test]
+    fn test_can_extract_djot_mime_types() {
+        let extractor = DjotExtractor::new();
+        let mime_types = extractor.supported_mime_types();
+
+        assert!(mime_types.contains(&"text/djot"));
+        assert!(mime_types.contains(&"text/x-djot"));
+    }
+
+    #[test]
+    fn test_plugin_interface() {
+        let extractor = DjotExtractor::new();
+        assert_eq!(extractor.author(), "Kreuzberg Team");
+        assert!(!extractor.version().is_empty());
+        assert!(!extractor.description().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_extract_simple_djot() {
+        let content =
+            b"# Header\n\nThis is a paragraph with *bold* and _italic_ text.\n\n## Subheading\n\nMore content here.";
+        let extractor = DjotExtractor::new();
+        let config = ExtractionConfig::default();
+
+        let result = extractor.extract_bytes(content, "text/djot", &config).await;
+        assert!(result.is_ok());
+
+        let result = result.unwrap();
+        assert!(result.content.contains("Header"));
+        assert!(result.content.contains("This is a paragraph"));
+        assert!(result.content.contains("bold"));
+        assert!(result.content.contains("italic"));
+    }
+}

data/vendor/kreuzberg/src/extractors/djot_format/mod.rs

@@ -0,0 +1,24 @@
+//! Djot markup format extractor and utilities.
+//!
+//! This module provides:
+//! - Djot parsing using the jotdown crate
+//! - YAML frontmatter metadata extraction (same as Markdown)
+//! - Table extraction as structured data
+//! - Heading structure preservation
+//! - Code block and link extraction
+//! - Djot content rendering and conversion APIs
+//!
+//! Djot is a modern markup language with simpler parsing rules than CommonMark.
+//! See https://djot.net for the specification.
+//!
+//! Requires the `djot` feature.
+
+pub mod attributes;
+pub mod conversion;
+pub mod extractor;
+pub mod parsing;
+pub mod rendering;
+
+// Re-export public API
+pub use conversion::{djot_content_to_djot, djot_to_html, extraction_result_to_djot};
+pub use extractor::DjotExtractor;