kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/types/djot.rs

@@ -0,0 +1,209 @@
+//! Djot document types.
+//!
+//! This module defines types for representing Djot document structures.
+
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+// Import Metadata and Table from parent module
+use super::Table;
+use super::metadata::Metadata;
+
+/// Comprehensive Djot document structure with semantic preservation.
+///
+/// This type captures the full richness of Djot markup, including:
+/// - Block-level structures (headings, lists, blockquotes, code blocks, etc.)
+/// - Inline formatting (emphasis, strong, highlight, subscript, superscript, etc.)
+/// - Attributes (classes, IDs, key-value pairs)
+/// - Links, images, footnotes
+/// - Math expressions (inline and display)
+/// - Tables with full structure
+///
+/// Available when the `djot` feature is enabled.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DjotContent {
+    /// Plain text representation for backwards compatibility
+    pub plain_text: String,
+
+    /// Structured block-level content
+    pub blocks: Vec<FormattedBlock>,
+
+    /// Metadata from YAML frontmatter
+    pub metadata: Metadata,
+
+    /// Extracted tables as structured data
+    pub tables: Vec<Table>,
+
+    /// Extracted images with metadata
+    pub images: Vec<DjotImage>,
+
+    /// Extracted links with URLs
+    pub links: Vec<DjotLink>,
+
+    /// Footnote definitions
+    pub footnotes: Vec<Footnote>,
+
+    /// Attributes mapped by element identifier (if present)
+    #[serde(skip_serializing_if = "HashMap::is_empty", default)]
+    pub attributes: HashMap<String, Attributes>,
+}
+
+/// Block-level element in a Djot document.
+///
+/// Represents structural elements like headings, paragraphs, lists, code blocks, etc.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FormattedBlock {
+    /// Type of block element
+    pub block_type: BlockType,
+
+    /// Heading level (1-6) for headings, or nesting level for lists
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub level: Option<usize>,
+
+    /// Inline content within the block
+    pub inline_content: Vec<InlineElement>,
+
+    /// Element attributes (classes, IDs, key-value pairs)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub attributes: Option<Attributes>,
+
+    /// Language identifier for code blocks
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub language: Option<String>,
+
+    /// Raw code content for code blocks
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub code: Option<String>,
+
+    /// Nested blocks for containers (blockquotes, list items, divs)
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    pub children: Vec<FormattedBlock>,
+}
+
+/// Types of block-level elements in Djot.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum BlockType {
+    Paragraph,
+    Heading,
+    Blockquote,
+    CodeBlock,
+    ListItem,
+    OrderedList,
+    BulletList,
+    TaskList,
+    DefinitionList,
+    DefinitionTerm,
+    DefinitionDescription,
+    Div,
+    Section,
+    ThematicBreak,
+    RawBlock,
+    MathDisplay,
+}
+
+/// Inline element within a block.
+///
+/// Represents text with formatting, links, images, etc.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct InlineElement {
+    /// Type of inline element
+    pub element_type: InlineType,
+
+    /// Text content
+    pub content: String,
+
+    /// Element attributes
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub attributes: Option<Attributes>,
+
+    /// Additional metadata (e.g., href for links, src/alt for images)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub metadata: Option<HashMap<String, String>>,
+}
+
+/// Types of inline elements in Djot.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum InlineType {
+    Text,
+    Strong,
+    Emphasis,
+    Highlight,
+    Subscript,
+    Superscript,
+    Insert,
+    Delete,
+    Code,
+    Link,
+    Image,
+    Span,
+    Math,
+    RawInline,
+    FootnoteRef,
+    Symbol,
+}
+
+/// Element attributes in Djot.
+///
+/// Represents the attributes attached to elements using {.class #id key="value"} syntax.
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct Attributes {
+    /// Element ID (#identifier)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub id: Option<String>,
+
+    /// CSS classes (.class1 .class2)
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    pub classes: Vec<String>,
+
+    /// Key-value pairs (key="value")
+    #[serde(skip_serializing_if = "HashMap::is_empty", default)]
+    pub key_values: HashMap<String, String>,
+}
+
+/// Image element in Djot.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DjotImage {
+    /// Image source URL or path
+    pub src: String,
+
+    /// Alternative text
+    pub alt: String,
+
+    /// Optional title
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub title: Option<String>,
+
+    /// Element attributes
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub attributes: Option<Attributes>,
+}
+
+/// Link element in Djot.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DjotLink {
+    /// Link URL
+    pub url: String,
+
+    /// Link text content
+    pub text: String,
+
+    /// Optional title
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub title: Option<String>,
+
+    /// Element attributes
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub attributes: Option<Attributes>,
+}
+
+/// Footnote in Djot.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Footnote {
+    /// Footnote label
+    pub label: String,
+
+    /// Footnote content blocks
+    pub content: Vec<FormattedBlock>,
+}

data/vendor/kreuzberg/src/types/extraction.rs

@@ -0,0 +1,301 @@
+//! Core extraction types and results.
+
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+use super::djot::DjotContent;
+use super::metadata::Metadata;
+use super::page::PageContent;
+use super::tables::Table;
+
+/// General extraction result used by the core extraction API.
+///
+/// This is the main result type returned by all extraction functions.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ExtractionResult {
+    pub content: String,
+    pub mime_type: String,
+    pub metadata: Metadata,
+    pub tables: Vec<Table>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub detected_languages: Option<Vec<String>>,
+
+    /// Text chunks when chunking is enabled.
+    ///
+    /// When chunking configuration is provided, the content is split into
+    /// overlapping chunks for efficient processing. Each chunk contains the text,
+    /// optional embeddings (if enabled), and metadata about its position.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub chunks: Option<Vec<Chunk>>,
+
+    /// Extracted images from the document.
+    ///
+    /// When image extraction is enabled via `ImageExtractionConfig`, this field
+    /// contains all images found in the document with their raw data and metadata.
+    /// Each image may optionally contain a nested `ocr_result` if OCR was performed.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub images: Option<Vec<ExtractedImage>>,
+
+    /// Per-page content when page extraction is enabled.
+    ///
+    /// When page extraction is configured, the document is split into per-page content
+    /// with tables and images mapped to their respective pages.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pages: Option<Vec<PageContent>>,
+
+    /// Semantic elements when element-based output format is enabled.
+    ///
+    /// When output_format is set to ElementBased, this field contains semantic
+    /// elements with type classification, unique identifiers, and metadata for
+    /// Unstructured-compatible element-based processing.
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    pub elements: Option<Vec<Element>>,
+
+    /// Rich Djot content structure (when extracting Djot documents).
+    ///
+    /// When extracting Djot documents with structured extraction enabled,
+    /// this field contains the full semantic structure including:
+    /// - Block-level elements with nesting
+    /// - Inline formatting with attributes
+    /// - Links, images, footnotes
+    /// - Math expressions
+    /// - Complete attribute information
+    ///
+    /// The `content` field still contains plain text for backward compatibility.
+    ///
+    /// Always `None` for non-Djot documents.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
+    pub djot_content: Option<DjotContent>,
+}
+
+/// A text chunk with optional embedding and metadata.
+///
+/// Chunks are created when chunking is enabled in `ExtractionConfig`. Each chunk
+/// contains the text content, optional embedding vector (if embedding generation
+/// is configured), and metadata about its position in the document.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Chunk {
+    /// The text content of this chunk.
+    pub content: String,
+
+    /// Optional embedding vector for this chunk.
+    ///
+    /// Only populated when `EmbeddingConfig` is provided in chunking configuration.
+    /// The dimensionality depends on the chosen embedding model.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub embedding: Option<Vec<f32>>,
+
+    /// Metadata about this chunk's position and properties.
+    pub metadata: ChunkMetadata,
+}
+
+/// Metadata about a chunk's position in the original document.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ChunkMetadata {
+    /// Byte offset where this chunk starts in the original text (UTF-8 valid boundary).
+    pub byte_start: usize,
+
+    /// Byte offset where this chunk ends in the original text (UTF-8 valid boundary).
+    pub byte_end: usize,
+
+    /// Number of tokens in this chunk (if available).
+    ///
+    /// This is calculated by the embedding model's tokenizer if embeddings are enabled.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub token_count: Option<usize>,
+
+    /// Zero-based index of this chunk in the document.
+    pub chunk_index: usize,
+
+    /// Total number of chunks in the document.
+    pub total_chunks: usize,
+
+    /// First page number this chunk spans (1-indexed).
+    ///
+    /// Only populated when page tracking is enabled in extraction configuration.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub first_page: Option<usize>,
+
+    /// Last page number this chunk spans (1-indexed, equal to first_page for single-page chunks).
+    ///
+    /// Only populated when page tracking is enabled in extraction configuration.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub last_page: Option<usize>,
+}
+
+/// Extracted image from a document.
+///
+/// Contains raw image data, metadata, and optional nested OCR results.
+/// Raw bytes allow cross-language compatibility - users can convert to
+/// PIL.Image (Python), Sharp (Node.js), or other formats as needed.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ExtractedImage {
+    /// Raw image data (PNG, JPEG, WebP, etc. bytes)
+    pub data: Vec<u8>,
+
+    /// Image format (e.g., "jpeg", "png", "webp")
+    pub format: String,
+
+    /// Zero-indexed position of this image in the document/page
+    pub image_index: usize,
+
+    /// Page/slide number where image was found (1-indexed)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub page_number: Option<usize>,
+
+    /// Image width in pixels
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub width: Option<u32>,
+
+    /// Image height in pixels
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub height: Option<u32>,
+
+    /// Colorspace information (e.g., "RGB", "CMYK", "Gray")
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub colorspace: Option<String>,
+
+    /// Bits per color component (e.g., 8, 16)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub bits_per_component: Option<u32>,
+
+    /// Whether this image is a mask image
+    #[serde(default)]
+    pub is_mask: bool,
+
+    /// Optional description of the image
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
+
+    /// Nested OCR extraction result (if image was OCRed)
+    ///
+    /// When OCR is performed on this image, the result is embedded here
+    /// rather than in a separate collection, making the relationship explicit.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ocr_result: Option<Box<ExtractionResult>>,
+}
+
+// ============================================================================
+// Element-based Output Format Types (Unstructured-compatible)
+// ============================================================================
+
+/// Output format selection for extraction results.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
+#[serde(rename_all = "snake_case")]
+pub enum OutputFormat {
+    /// Unified format with all content in `content` field
+    #[default]
+    Unified,
+    /// Element-based format with semantic element extraction
+    ElementBased,
+}
+
+/// Unique identifier for semantic elements.
+///
+/// Wraps a string identifier that is deterministically generated
+/// from element type, content, and page number.
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct ElementId(String);
+
+impl ElementId {
+    /// Create a new ElementId from a string.
+    ///
+    /// # Errors
+    ///
+    /// Returns error if the string is not valid.
+    pub fn new(hex_str: impl Into<String>) -> std::result::Result<Self, String> {
+        let s = hex_str.into();
+        if s.is_empty() {
+            return Err("ElementId cannot be empty".to_string());
+        }
+        Ok(ElementId(s))
+    }
+}
+
+impl AsRef<str> for ElementId {
+    fn as_ref(&self) -> &str {
+        &self.0
+    }
+}
+
+impl std::fmt::Display for ElementId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+/// Semantic element type classification.
+///
+/// Categorizes text content into semantic units for downstream processing.
+/// Supports the element types commonly found in Unstructured documents.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ElementType {
+    /// Document title
+    Title,
+    /// Main narrative text body
+    NarrativeText,
+    /// Section heading
+    Heading,
+    /// List item (bullet, numbered, etc.)
+    ListItem,
+    /// Table element
+    Table,
+    /// Image element
+    Image,
+    /// Page break marker
+    PageBreak,
+    /// Code block
+    CodeBlock,
+    /// Block quote
+    BlockQuote,
+    /// Footer text
+    Footer,
+    /// Header text
+    Header,
+}
+
+/// Bounding box coordinates for element positioning.
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct BoundingBox {
+    /// Left x-coordinate
+    pub x0: f64,
+    /// Bottom y-coordinate
+    pub y0: f64,
+    /// Right x-coordinate
+    pub x1: f64,
+    /// Top y-coordinate
+    pub y1: f64,
+}
+
+/// Metadata for a semantic element.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ElementMetadata {
+    /// Page number (1-indexed)
+    pub page_number: Option<usize>,
+    /// Source filename or document name
+    pub filename: Option<String>,
+    /// Bounding box coordinates if available
+    pub coordinates: Option<BoundingBox>,
+    /// Position index in the element sequence
+    pub element_index: Option<usize>,
+    /// Additional custom metadata
+    pub additional: HashMap<String, String>,
+}
+
+/// Semantic element extracted from document.
+///
+/// Represents a logical unit of content with semantic classification,
+/// unique identifier, and metadata for tracking origin and position.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Element {
+    /// Unique element identifier
+    pub element_id: ElementId,
+    /// Semantic type of this element
+    pub element_type: ElementType,
+    /// Text content of the element
+    pub text: String,
+    /// Metadata about the element
+    pub metadata: ElementMetadata,
+}