kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/extractors/djot_format/parsing/block_handlers.rs

@@ -0,0 +1,271 @@
+//! Block-level container handlers for Djot parsing.
+
+use super::state::{ExtractionState, push_block};
+use crate::types::{Attributes, BlockType, FormattedBlock};
+use jotdown::Container;
+
+/// Handle start of block containers.
+pub(super) fn handle_block_start(
+    state: &mut ExtractionState,
+    container: &Container,
+    _attrs: &jotdown::Attributes,
+    parsed_attrs: Option<Attributes>,
+    footnotes: &mut Vec<crate::types::Footnote>,
+) -> bool {
+    match container {
+        Container::Heading { level, .. } => {
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::Heading,
+                    level: Some(*level as usize),
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::Paragraph => {
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::Paragraph,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::Blockquote => {
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::Blockquote,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::CodeBlock { language } => {
+            let lang_str = if language.is_empty() {
+                None
+            } else {
+                Some(language.to_string())
+            };
+            state.in_code_block = true;
+            state.code_content.clear();
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::CodeBlock,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: lang_str,
+                    code: Some(String::new()),
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::RawBlock { format } => {
+            state.in_raw_block = true;
+            state.raw_format = Some(format.to_string());
+            state.code_content.clear();
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::RawBlock,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: Some(format.to_string()),
+                    code: Some(String::new()),
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::List { kind, .. } => {
+            let block_type = match kind {
+                jotdown::ListKind::Ordered { .. } => BlockType::OrderedList,
+                jotdown::ListKind::Unordered(_) => BlockType::BulletList,
+                jotdown::ListKind::Task(_) => BlockType::TaskList,
+            };
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::ListItem => {
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::ListItem,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::TaskListItem { checked } => {
+            let mut attrs = parsed_attrs.unwrap_or_default();
+            attrs.key_values.insert("checked".to_string(), checked.to_string());
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::ListItem,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: Some(attrs),
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::DescriptionList => {
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::DefinitionList,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::DescriptionTerm => {
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::DefinitionTerm,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::DescriptionDetails => {
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::DefinitionDescription,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::Div { .. } => {
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::Div,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::Section { .. } => {
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::Section,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        Container::Footnote { label } => {
+            // Start tracking a footnote definition
+            footnotes.push(crate::types::Footnote {
+                label: label.to_string(),
+                content: Vec::new(),
+            });
+            // We'll collect the content as blocks
+            push_block(
+                state,
+                FormattedBlock {
+                    block_type: BlockType::Paragraph,
+                    level: None,
+                    inline_content: Vec::new(),
+                    attributes: parsed_attrs,
+                    language: None,
+                    code: None,
+                    children: Vec::new(),
+                },
+            );
+            true
+        }
+        _ => false,
+    }
+}
+
+/// Handle end of block containers.
+pub(super) fn handle_block_end(_state: &mut ExtractionState, container: &Container) -> bool {
+    matches!(
+        container,
+        Container::Heading { .. }
+            | Container::Paragraph
+            | Container::Blockquote
+            | Container::CodeBlock { .. }
+            | Container::RawBlock { .. }
+            | Container::Div { .. }
+            | Container::Section { .. }
+            | Container::List { .. }
+            | Container::ListItem
+            | Container::TaskListItem { .. }
+            | Container::DescriptionList
+            | Container::DescriptionTerm
+            | Container::DescriptionDetails
+    )
+}

data/vendor/kreuzberg/src/extractors/djot_format/parsing/content_extraction.rs

@@ -0,0 +1,257 @@
+//! Complete Djot content extraction.
+//!
+//! Handles extraction of rich DjotContent structures from Djot events.
+
+use super::block_handlers::{handle_block_end, handle_block_start};
+use super::event_handlers::{
+    finalize_block_element, handle_footnote_end, handle_footnote_reference, handle_symbol, handle_thematic_break,
+};
+use super::inline_handlers::{
+    finalize_inline_element, handle_image_end, handle_inline_end, handle_inline_start, handle_link_end, handle_math_end,
+};
+use super::state::{ExtractionState, pop_block};
+use super::text_extraction::extract_text_from_events;
+use crate::extractors::djot_format::attributes::parse_jotdown_attributes;
+use crate::types::{Attributes, DjotContent, DjotImage, DjotLink, FormattedBlock};
+use jotdown::{Container, Event};
+use std::collections::HashMap;
+
+/// Extract complete djot content with 100% feature extraction.
+///
+/// Processes ALL djot events to build a rich DjotContent structure including:
+/// - Block structure (headings, lists, blockquotes, divs, sections, code blocks)
+/// - Inline formatting (strong, emphasis, highlight, subscript, superscript, insert, delete)
+/// - Attributes (classes, IDs, key-value pairs)
+/// - Links and images with full metadata (href, src, alt, title)
+/// - Math blocks (inline & display)
+/// - Definition lists (term/description pairs)
+/// - Task lists with checked state
+/// - Raw blocks (HTML/LaTeX)
+/// - Footnotes (references and definitions)
+/// - Captions
+/// - Smart punctuation
+/// - All other djot features
+pub fn extract_complete_djot_content(
+    events: &[Event],
+    metadata: crate::types::Metadata,
+    tables: Vec<crate::types::Table>,
+) -> DjotContent {
+    let plain_text = extract_text_from_events(events);
+
+    let mut blocks = Vec::new();
+    let mut images = Vec::new();
+    let mut links = Vec::new();
+    let mut footnotes = Vec::new();
+    let attributes_map: HashMap<String, Attributes> = HashMap::new();
+
+    let mut state = ExtractionState::new();
+
+    for event in events {
+        match event {
+            Event::Start(container, attrs) => {
+                handle_start_event(
+                    &mut state,
+                    container,
+                    attrs,
+                    &mut blocks,
+                    &mut images,
+                    &mut links,
+                    &mut footnotes,
+                );
+            }
+            Event::End(container) => {
+                handle_end_event(
+                    &mut state,
+                    container,
+                    &mut blocks,
+                    &mut images,
+                    &mut links,
+                    &mut footnotes,
+                );
+            }
+            Event::Str(s) => {
+                if state.in_code_block || state.in_raw_block {
+                    state.code_content.push_str(s);
+                } else if state.in_math {
+                    state.math_content.push_str(s);
+                } else {
+                    state.current_text.push_str(s);
+                }
+            }
+            Event::FootnoteReference(label) => {
+                handle_footnote_reference(&mut state, label);
+            }
+            Event::Symbol(sym) => {
+                handle_symbol(&mut state, sym);
+            }
+            Event::Attributes(attrs) => {
+                // Store attributes to be applied to the next element
+                state.pending_attributes = Some(parse_jotdown_attributes(attrs));
+            }
+            Event::Softbreak => {
+                if state.in_math {
+                    state.math_content.push(' ');
+                } else if !state.inline_type_stack.is_empty() {
+                    state.current_text.push(' ');
+                } else {
+                    state.current_text.push('\n');
+                }
+            }
+            Event::Hardbreak => {
+                if state.in_math {
+                    state.math_content.push('\n');
+                } else {
+                    state.current_text.push('\n');
+                }
+            }
+            Event::NonBreakingSpace => {
+                state.current_text.push(' ');
+            }
+            Event::Blankline => {
+                // Blank lines are typically ignored in block processing
+            }
+            Event::ThematicBreak(attrs) => {
+                handle_thematic_break(&mut state, attrs, &mut blocks);
+            }
+            // Smart punctuation events
+            Event::LeftSingleQuote => {
+                state.current_text.push('\'');
+            }
+            Event::RightSingleQuote => {
+                state.current_text.push('\'');
+            }
+            Event::LeftDoubleQuote => {
+                state.current_text.push('"');
+            }
+            Event::RightDoubleQuote => {
+                state.current_text.push('"');
+            }
+            Event::Ellipsis => {
+                state.current_text.push_str("...");
+            }
+            Event::EnDash => {
+                state.current_text.push_str("--");
+            }
+            Event::EmDash => {
+                state.current_text.push_str("---");
+            }
+            Event::Escape => {
+                // Escape is a marker, doesn't produce output
+            }
+        }
+    }
+
+    // Finalize any remaining content
+    state.flush_text();
+
+    // Pop any remaining blocks
+    while !state.block_stack.is_empty() {
+        pop_block(&mut state, &mut blocks);
+    }
+
+    // Add any remaining inline elements to the last block if exists
+    if !state.current_inline_elements.is_empty()
+        && let Some(last_block) = blocks.last_mut()
+    {
+        last_block.inline_content.append(&mut state.current_inline_elements);
+    }
+
+    DjotContent {
+        plain_text,
+        blocks,
+        metadata,
+        tables,
+        images,
+        links,
+        footnotes,
+        attributes: attributes_map,
+    }
+}
+
+/// Handle start of a container event.
+fn handle_start_event(
+    state: &mut ExtractionState,
+    container: &Container,
+    attrs: &jotdown::Attributes,
+    _blocks: &mut Vec<FormattedBlock>,
+    images: &mut Vec<DjotImage>,
+    links: &mut Vec<DjotLink>,
+    footnotes: &mut Vec<crate::types::Footnote>,
+) {
+    // Parse attributes from jotdown's Attributes type
+    let parsed_attrs = if attrs.is_empty() {
+        state.pending_attributes.take()
+    } else {
+        Some(parse_jotdown_attributes(attrs))
+    };
+
+    // Try block handlers first
+    if handle_block_start(state, container, attrs, parsed_attrs.clone(), footnotes) {
+        return;
+    }
+
+    // Try inline handlers
+    if handle_inline_start(state, container, parsed_attrs, images, links) {
+        return;
+    }
+
+    // Handle remaining containers (tables, link definitions, etc.)
+    match container {
+        Container::Table | Container::TableRow { .. } | Container::TableCell { .. } | Container::Caption => {
+            // Tables are extracted separately
+        }
+        Container::LinkDefinition { .. } => {
+            // Link definitions are resolved by jotdown, not needed in output
+        }
+        _ => {}
+    }
+}
+
+/// Handle end of a container event.
+fn handle_end_event(
+    state: &mut ExtractionState,
+    container: &Container,
+    blocks: &mut Vec<FormattedBlock>,
+    images: &mut [DjotImage],
+    links: &mut [DjotLink],
+    footnotes: &mut [crate::types::Footnote],
+) {
+    // Check if it's a block container
+    if handle_block_end(state, container) {
+        finalize_block_element(state, blocks);
+        return;
+    }
+
+    // Handle special cases
+    match container {
+        Container::Footnote { .. } => {
+            handle_footnote_end(state, footnotes);
+        }
+        Container::Math { display } => {
+            handle_math_end(state, *display);
+        }
+        Container::Link(url, _) => {
+            handle_link_end(state, url, links);
+        }
+        Container::Image(src, _) => {
+            handle_image_end(state, src, images);
+        }
+        _ => {
+            // Check if it's an inline element
+            if handle_inline_end(state, container) {
+                finalize_inline_element(state, container);
+            }
+        }
+    }
+
+    // Handle remaining containers (tables, link definitions, etc.)
+    match container {
+        Container::Table | Container::TableRow { .. } | Container::TableCell { .. } | Container::Caption => {
+            // Tables are handled separately
+        }
+        Container::LinkDefinition { .. } => {
+            // Link definitions don't produce output
+        }
+        _ => {}
+    }
+}

data/vendor/kreuzberg/src/extractors/djot_format/parsing/event_handlers.rs

@@ -0,0 +1,101 @@
+//! Event handlers for special Djot elements.
+
+use super::state::{ExtractionState, pop_block};
+use crate::extractors::djot_format::attributes::parse_jotdown_attributes;
+use crate::types::{BlockType, FormattedBlock, InlineElement, InlineType};
+use std::collections::HashMap;
+
+/// Handle footnote reference event.
+pub(super) fn handle_footnote_reference(state: &mut ExtractionState, label: &str) {
+    state.flush_text();
+
+    let mut meta = HashMap::new();
+    meta.insert("label".to_string(), label.to_string());
+
+    state.current_inline_elements.push(InlineElement {
+        element_type: InlineType::FootnoteRef,
+        content: label.to_string(),
+        attributes: None,
+        metadata: Some(meta),
+    });
+}
+
+/// Handle symbol event.
+pub(super) fn handle_symbol(state: &mut ExtractionState, sym: &str) {
+    state.flush_text();
+
+    state.current_inline_elements.push(InlineElement {
+        element_type: InlineType::Symbol,
+        content: sym.to_string(),
+        attributes: None,
+        metadata: None,
+    });
+}
+
+/// Handle thematic break event.
+pub(super) fn handle_thematic_break(
+    state: &mut ExtractionState,
+    attrs: &jotdown::Attributes,
+    blocks: &mut Vec<FormattedBlock>,
+) {
+    state.flush_text();
+
+    let parsed_attrs = if attrs.is_empty() {
+        None
+    } else {
+        Some(parse_jotdown_attributes(attrs))
+    };
+
+    let hr_block = FormattedBlock {
+        block_type: BlockType::ThematicBreak,
+        level: None,
+        inline_content: Vec::new(),
+        attributes: parsed_attrs,
+        language: None,
+        code: None,
+        children: Vec::new(),
+    };
+
+    if let Some(parent) = state.block_stack.last_mut() {
+        parent.children.push(hr_block);
+    } else {
+        blocks.push(hr_block);
+    }
+}
+
+/// Handle end of footnote definition.
+pub(super) fn handle_footnote_end(state: &mut ExtractionState, footnotes: &mut [crate::types::Footnote]) {
+    state.flush_text();
+    // Pop the footnote content block and add to the last footnote
+    if let Some(mut block) = state.block_stack.pop() {
+        block.inline_content.append(&mut state.current_inline_elements);
+        if let Some(footnote) = footnotes.last_mut() {
+            footnote.content.push(block);
+        }
+    }
+}
+
+/// Finalize block element content and pop from stack.
+pub(super) fn finalize_block_element(state: &mut ExtractionState, blocks: &mut Vec<FormattedBlock>) {
+    // Flush any remaining text
+    state.flush_text();
+
+    // For code blocks, set the accumulated code content
+    if state.in_code_block {
+        if let Some(block) = state.block_stack.last_mut() {
+            block.code = Some(std::mem::take(&mut state.code_content));
+        }
+        state.in_code_block = false;
+    }
+
+    // For raw blocks
+    if state.in_raw_block {
+        if let Some(block) = state.block_stack.last_mut() {
+            block.code = Some(std::mem::take(&mut state.code_content));
+        }
+        state.in_raw_block = false;
+        state.raw_format = None;
+    }
+
+    pop_block(state, blocks);
+}