kreuzberg 4.0.7 → 4.1.0

data/vendor/kreuzberg/src/extractors/djot_format/parsing/inline_handlers.rs

@@ -0,0 +1,201 @@
+//! Inline-level element handlers for Djot parsing.
+
+use super::state::ExtractionState;
+use crate::types::{DjotImage, DjotLink, InlineElement, InlineType};
+use jotdown::Container;
+use std::collections::HashMap;
+
+/// Handle start of inline elements.
+pub(super) fn handle_inline_start(
+    state: &mut ExtractionState,
+    container: &Container,
+    parsed_attrs: Option<crate::types::Attributes>,
+    images: &mut Vec<DjotImage>,
+    links: &mut Vec<DjotLink>,
+) -> bool {
+    match container {
+        Container::Math { display } => {
+            state.in_math = true;
+            state.math_display = *display;
+            state.math_content.clear();
+            state.inline_type_stack.push(InlineType::Math);
+            true
+        }
+        Container::Strong => {
+            state.inline_type_stack.push(InlineType::Strong);
+            state.flush_text();
+            true
+        }
+        Container::Emphasis => {
+            state.inline_type_stack.push(InlineType::Emphasis);
+            state.flush_text();
+            true
+        }
+        Container::Mark => {
+            state.inline_type_stack.push(InlineType::Highlight);
+            state.flush_text();
+            true
+        }
+        Container::Subscript => {
+            state.inline_type_stack.push(InlineType::Subscript);
+            state.flush_text();
+            true
+        }
+        Container::Superscript => {
+            state.inline_type_stack.push(InlineType::Superscript);
+            state.flush_text();
+            true
+        }
+        Container::Insert => {
+            state.inline_type_stack.push(InlineType::Insert);
+            state.flush_text();
+            true
+        }
+        Container::Delete => {
+            state.inline_type_stack.push(InlineType::Delete);
+            state.flush_text();
+            true
+        }
+        Container::Verbatim => {
+            state.inline_type_stack.push(InlineType::Code);
+            state.flush_text();
+            true
+        }
+        Container::Span => {
+            state.inline_type_stack.push(InlineType::Span);
+            state.flush_text();
+            true
+        }
+        Container::RawInline { format } => {
+            state.inline_type_stack.push(InlineType::RawInline);
+            state.raw_format = Some(format.to_string());
+            state.flush_text();
+            true
+        }
+        Container::Link(url, _link_type) => {
+            state.inline_type_stack.push(InlineType::Link);
+            links.push(DjotLink {
+                url: url.to_string(),
+                text: String::new(),
+                title: None,
+                attributes: parsed_attrs,
+            });
+            state.current_link_index = Some(links.len() - 1);
+            state.flush_text();
+            true
+        }
+        Container::Image(src, _link_type) => {
+            state.inline_type_stack.push(InlineType::Image);
+            images.push(DjotImage {
+                src: src.to_string(),
+                alt: String::new(),
+                title: None,
+                attributes: parsed_attrs,
+            });
+            state.current_image_index = Some(images.len() - 1);
+            state.flush_text();
+            true
+        }
+        _ => false,
+    }
+}
+
+/// Handle end of inline elements.
+pub(super) fn handle_inline_end(_state: &mut ExtractionState, container: &Container) -> bool {
+    matches!(
+        container,
+        Container::Strong
+            | Container::Emphasis
+            | Container::Mark
+            | Container::Subscript
+            | Container::Superscript
+            | Container::Insert
+            | Container::Delete
+            | Container::Verbatim
+            | Container::Span
+            | Container::RawInline { .. }
+    )
+}
+
+/// Handle end of math element.
+pub(super) fn handle_math_end(state: &mut ExtractionState, display: bool) {
+    state.in_math = false;
+    let math_text = std::mem::take(&mut state.math_content);
+    state.inline_type_stack.pop();
+
+    let mut meta = HashMap::new();
+    meta.insert("display".to_string(), display.to_string());
+
+    state.current_inline_elements.push(InlineElement {
+        element_type: InlineType::Math,
+        content: math_text,
+        attributes: state.pending_attributes.take(),
+        metadata: Some(meta),
+    });
+}
+
+/// Finalize inline element content.
+pub(super) fn finalize_inline_element(state: &mut ExtractionState, container: &Container) {
+    if let Some(inline_type) = state.inline_type_stack.pop() {
+        let content = std::mem::take(&mut state.current_text);
+        let mut meta = None;
+
+        // For raw inline, include the format
+        if matches!(container, Container::RawInline { .. })
+            && let Some(fmt) = state.raw_format.take()
+        {
+            let mut m = HashMap::new();
+            m.insert("format".to_string(), fmt);
+            meta = Some(m);
+        }
+
+        state.current_inline_elements.push(InlineElement {
+            element_type: inline_type,
+            content,
+            attributes: state.pending_attributes.take(),
+            metadata: meta,
+        });
+    }
+}
+
+/// Handle end of link element.
+pub(super) fn handle_link_end(state: &mut ExtractionState, url: &str, links: &mut [DjotLink]) {
+    if let Some(idx) = state.current_link_index.take() {
+        let text = std::mem::take(&mut state.current_text);
+        if let Some(link) = links.get_mut(idx) {
+            link.text = text.clone();
+        }
+        state.inline_type_stack.pop();
+
+        let mut meta = HashMap::new();
+        meta.insert("href".to_string(), url.to_string());
+
+        state.current_inline_elements.push(InlineElement {
+            element_type: InlineType::Link,
+            content: text,
+            attributes: state.pending_attributes.take(),
+            metadata: Some(meta),
+        });
+    }
+}
+
+/// Handle end of image element.
+pub(super) fn handle_image_end(state: &mut ExtractionState, src: &str, images: &mut [DjotImage]) {
+    if let Some(idx) = state.current_image_index.take() {
+        let alt = std::mem::take(&mut state.current_text);
+        if let Some(image) = images.get_mut(idx) {
+            image.alt = alt.clone();
+        }
+        state.inline_type_stack.pop();
+
+        let mut meta = HashMap::new();
+        meta.insert("src".to_string(), src.to_string());
+
+        state.current_inline_elements.push(InlineElement {
+            element_type: InlineType::Image,
+            content: alt,
+            attributes: state.pending_attributes.take(),
+            metadata: Some(meta),
+        });
+    }
+}

data/vendor/kreuzberg/src/extractors/djot_format/parsing/mod.rs

@@ -0,0 +1,16 @@
+//! Djot event parsing and content extraction.
+//!
+//! Handles parsing of jotdown events into plain text, tables, and full DjotContent structures.
+
+mod block_handlers;
+mod content_extraction;
+mod event_handlers;
+mod inline_handlers;
+mod state;
+mod table_extraction;
+mod text_extraction;
+
+// Re-export public API for backward compatibility
+pub use content_extraction::extract_complete_djot_content;
+pub use table_extraction::extract_tables_from_events;
+pub use text_extraction::extract_text_from_events;

data/vendor/kreuzberg/src/extractors/djot_format/parsing/state.rs

@@ -0,0 +1,78 @@
+//! State management for Djot content extraction.
+//!
+//! Provides extraction state and helper functions for parsing Djot events.
+
+use crate::types::{Attributes, FormattedBlock, InlineElement, InlineType};
+
+/// Enhanced state tracking using a block stack for proper nesting.
+pub(super) struct ExtractionState {
+    pub block_stack: Vec<FormattedBlock>,   // Stack for nested blocks
+    pub inline_type_stack: Vec<InlineType>, // Stack for nested inline element types
+    pub current_text: String,               // Text accumulator
+    pub pending_attributes: Option<Attributes>,
+    pub code_content: String, // Accumulator for code blocks
+    pub in_code_block: bool,
+    pub in_math: bool,
+    pub math_display: bool,
+    pub math_content: String,
+    pub current_link_index: Option<usize>,
+    pub current_image_index: Option<usize>,
+    pub in_raw_block: bool,
+    pub raw_format: Option<String>,
+    pub current_inline_elements: Vec<InlineElement>,
+}
+
+impl ExtractionState {
+    /// Create a new extraction state.
+    pub fn new() -> Self {
+        Self {
+            block_stack: Vec::new(),
+            inline_type_stack: Vec::new(),
+            current_text: String::new(),
+            pending_attributes: None,
+            code_content: String::new(),
+            in_code_block: false,
+            in_math: false,
+            math_display: false,
+            math_content: String::new(),
+            current_link_index: None,
+            current_image_index: None,
+            in_raw_block: false,
+            raw_format: None,
+            current_inline_elements: Vec::new(),
+        }
+    }
+
+    /// Flush current text to inline elements if any text is pending.
+    pub fn flush_text(&mut self) {
+        if !self.current_text.is_empty() {
+            self.current_inline_elements.push(InlineElement {
+                element_type: InlineType::Text,
+                content: std::mem::take(&mut self.current_text),
+                attributes: None,
+                metadata: None,
+            });
+        }
+    }
+}
+
+/// Helper to create a new block and push to stack.
+pub(super) fn push_block(state: &mut ExtractionState, block: FormattedBlock) {
+    state.block_stack.push(block);
+}
+
+/// Helper to pop a block from the stack and add to parent or blocks list.
+pub(super) fn pop_block(state: &mut ExtractionState, blocks: &mut Vec<FormattedBlock>) {
+    if let Some(mut block) = state.block_stack.pop() {
+        // Add any pending inline elements to the block
+        if !state.current_inline_elements.is_empty() {
+            block.inline_content.append(&mut state.current_inline_elements);
+        }
+        // If there's a parent block, add as child; otherwise add to top-level blocks
+        if let Some(parent) = state.block_stack.last_mut() {
+            parent.children.push(block);
+        } else {
+            blocks.push(block);
+        }
+    }
+}

data/vendor/kreuzberg/src/extractors/djot_format/parsing/table_extraction.rs

@@ -0,0 +1,68 @@
+//! Table extraction from Djot events.
+//!
+//! Parses table events and extracts table data.
+
+use crate::types::Table;
+use jotdown::{Container, Event};
+
+/// Extract tables from Djot events.
+///
+/// Parses table events and extracts table data as a Vec<Vec<String>>,
+/// converting each table to markdown representation for storage.
+pub fn extract_tables_from_events(events: &[Event]) -> Vec<Table> {
+    let mut tables = Vec::new();
+    let mut current_table: Option<(Vec<Vec<String>>, usize)> = None;
+    let mut current_row: Vec<String> = Vec::new();
+    let mut current_cell = String::new();
+    let mut in_table_cell = false;
+    let mut table_index = 0;
+
+    for event in events {
+        match event {
+            Event::Start(Container::Table, _) => {
+                current_table = Some((Vec::new(), table_index));
+            }
+            Event::Start(Container::TableRow { .. }, _) => {
+                current_row = Vec::new();
+            }
+            Event::Start(Container::TableCell { .. }, _) => {
+                current_cell = String::new();
+                in_table_cell = true;
+            }
+            Event::Str(s) if in_table_cell => {
+                current_cell.push_str(s.as_ref());
+            }
+            Event::End(Container::TableCell { .. }) => {
+                if in_table_cell {
+                    current_row.push(current_cell.trim().to_string());
+                    current_cell = String::new();
+                    in_table_cell = false;
+                }
+            }
+            Event::End(Container::TableRow { .. }) => {
+                if !current_row.is_empty()
+                    && let Some((ref mut rows, _)) = current_table
+                {
+                    rows.push(current_row.clone());
+                }
+                current_row = Vec::new();
+            }
+            Event::End(Container::Table) => {
+                if let Some((cells, idx)) = current_table.take()
+                    && !cells.is_empty()
+                {
+                    let markdown = crate::extractors::frontmatter_utils::cells_to_markdown(&cells);
+                    tables.push(Table {
+                        cells,
+                        markdown,
+                        page_number: idx + 1,
+                    });
+                    table_index += 1;
+                }
+            }
+            _ => {}
+        }
+    }
+
+    tables
+}

data/vendor/kreuzberg/src/extractors/djot_format/parsing/text_extraction.rs

@@ -0,0 +1,61 @@
+//! Text extraction from Djot events.
+//!
+//! Handles parsing of jotdown events into plain text.
+
+use jotdown::Event;
+
+/// Extract plain text from Djot events.
+///
+/// Processes djot events and extracts plain text content, handling:
+/// - Text content
+/// - Line breaks (soft, hard, blank)
+/// - Smart punctuation (quotes, dashes, ellipsis)
+/// - Special symbols and footnote references
+pub fn extract_text_from_events(events: &[Event]) -> String {
+    let mut text = String::new();
+
+    for event in events {
+        match event {
+            Event::Str(s) => {
+                text.push_str(s.as_ref());
+            }
+            Event::Softbreak | Event::Hardbreak | Event::Blankline => {
+                text.push('\n');
+            }
+            Event::NonBreakingSpace => {
+                text.push(' ');
+            }
+            Event::LeftSingleQuote | Event::RightSingleQuote => {
+                text.push('\'');
+            }
+            Event::LeftDoubleQuote | Event::RightDoubleQuote => {
+                text.push('"');
+            }
+            Event::Ellipsis => {
+                text.push_str("...");
+            }
+            Event::EnDash => {
+                text.push_str("--");
+            }
+            Event::EmDash => {
+                text.push_str("---");
+            }
+            Event::FootnoteReference(s) => {
+                text.push('[');
+                text.push_str(s.as_ref());
+                text.push(']');
+            }
+            Event::Symbol(s) => {
+                text.push(':');
+                text.push_str(s.as_ref());
+                text.push(':');
+            }
+            Event::ThematicBreak(_) => {
+                text.push_str("\n---\n");
+            }
+            Event::Start(_, _) | Event::End(_) | Event::Escape | Event::Attributes(_) => {}
+        }
+    }
+
+    text
+}