kreuzberg 4.0.0.rc1

data/vendor/kreuzberg/src/pdf/error.rs

@@ -0,0 +1,122 @@
+use std::fmt;
+
+#[derive(Debug, Clone)]
+pub enum PdfError {
+    InvalidPdf(String),
+    PasswordRequired,
+    InvalidPassword,
+    EncryptionNotSupported(String),
+    PageNotFound(usize),
+    TextExtractionFailed(String),
+    RenderingFailed(String),
+    MetadataExtractionFailed(String),
+    IOError(String),
+}
+
+impl fmt::Display for PdfError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            PdfError::InvalidPdf(msg) => write!(f, "Invalid PDF: {}", msg),
+            PdfError::PasswordRequired => write!(f, "PDF is password-protected"),
+            PdfError::InvalidPassword => write!(f, "Invalid password provided"),
+            PdfError::EncryptionNotSupported(msg) => {
+                write!(f, "Encryption not supported: {}", msg)
+            }
+            PdfError::PageNotFound(page) => write!(f, "Page {} not found", page),
+            PdfError::TextExtractionFailed(msg) => write!(f, "Text extraction failed: {}", msg),
+            PdfError::RenderingFailed(msg) => write!(f, "Page rendering failed: {}", msg),
+            PdfError::MetadataExtractionFailed(msg) => {
+                write!(f, "Metadata extraction failed: {}", msg)
+            }
+            PdfError::IOError(msg) => write!(f, "I/O error: {}", msg),
+        }
+    }
+}
+
+impl std::error::Error for PdfError {}
+
+// NOTE: No From<std::io::Error> impl - IO errors must bubble up unchanged per error handling policy
+
+impl From<lopdf::Error> for PdfError {
+    fn from(err: lopdf::Error) -> Self {
+        match err {
+            lopdf::Error::IO(_) => panic!("lopdf IO errors should not be converted to PdfError - let them bubble up"),
+            _ => PdfError::InvalidPdf(err.to_string()),
+        }
+    }
+}
+
+pub type Result<T> = std::result::Result<T, PdfError>;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_invalid_pdf_error() {
+        let err = PdfError::InvalidPdf("corrupted header".to_string());
+        assert_eq!(err.to_string(), "Invalid PDF: corrupted header");
+    }
+
+    #[test]
+    fn test_password_required_error() {
+        let err = PdfError::PasswordRequired;
+        assert_eq!(err.to_string(), "PDF is password-protected");
+    }
+
+    #[test]
+    fn test_invalid_password_error() {
+        let err = PdfError::InvalidPassword;
+        assert_eq!(err.to_string(), "Invalid password provided");
+    }
+
+    #[test]
+    fn test_encryption_not_supported_error() {
+        let err = PdfError::EncryptionNotSupported("AES-256".to_string());
+        assert_eq!(err.to_string(), "Encryption not supported: AES-256");
+    }
+
+    #[test]
+    fn test_page_not_found_error() {
+        let err = PdfError::PageNotFound(5);
+        assert_eq!(err.to_string(), "Page 5 not found");
+    }
+
+    #[test]
+    fn test_text_extraction_failed_error() {
+        let err = PdfError::TextExtractionFailed("no text layer".to_string());
+        assert_eq!(err.to_string(), "Text extraction failed: no text layer");
+    }
+
+    #[test]
+    fn test_rendering_failed_error() {
+        let err = PdfError::RenderingFailed("out of memory".to_string());
+        assert_eq!(err.to_string(), "Page rendering failed: out of memory");
+    }
+
+    #[test]
+    fn test_metadata_extraction_failed_error() {
+        let err = PdfError::MetadataExtractionFailed("invalid metadata".to_string());
+        assert_eq!(err.to_string(), "Metadata extraction failed: invalid metadata");
+    }
+
+    #[test]
+    fn test_io_error() {
+        let err = PdfError::IOError("read failed".to_string());
+        assert_eq!(err.to_string(), "I/O error: read failed");
+    }
+
+    #[test]
+    fn test_error_debug() {
+        let err = PdfError::InvalidPassword;
+        let debug_str = format!("{:?}", err);
+        assert!(debug_str.contains("InvalidPassword"));
+    }
+
+    #[test]
+    fn test_error_clone() {
+        let err1 = PdfError::PageNotFound(3);
+        let err2 = err1.clone();
+        assert_eq!(err1.to_string(), err2.to_string());
+    }
+}

data/vendor/kreuzberg/src/pdf/images.rs

@@ -0,0 +1,139 @@
+use super::error::{PdfError, Result};
+use lopdf::Document;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PdfImage {
+    pub page_number: usize,
+    pub image_index: usize,
+    pub width: i64,
+    pub height: i64,
+    pub color_space: Option<String>,
+    pub bits_per_component: Option<i64>,
+    pub filters: Vec<String>,
+    pub data: Vec<u8>,
+}
+
+#[derive(Debug)]
+pub struct PdfImageExtractor {
+    document: Document,
+}
+
+impl PdfImageExtractor {
+    pub fn new(pdf_bytes: &[u8]) -> Result<Self> {
+        Self::new_with_password(pdf_bytes, None)
+    }
+
+    pub fn new_with_password(pdf_bytes: &[u8], password: Option<&str>) -> Result<Self> {
+        let mut doc =
+            Document::load_mem(pdf_bytes).map_err(|e| PdfError::InvalidPdf(format!("Failed to load PDF: {}", e)))?;
+
+        if doc.is_encrypted() {
+            if let Some(pwd) = password {
+                doc.decrypt(pwd).map_err(|_| PdfError::InvalidPassword)?;
+            } else {
+                return Err(PdfError::PasswordRequired);
+            }
+        }
+
+        Ok(Self { document: doc })
+    }
+
+    pub fn extract_images(&self) -> Result<Vec<PdfImage>> {
+        let mut all_images = Vec::new();
+        let pages = self.document.get_pages();
+
+        for (page_num, page_id) in pages.iter() {
+            let images = self
+                .document
+                .get_page_images(*page_id)
+                .map_err(|e| PdfError::MetadataExtractionFailed(format!("Failed to get page images: {}", e)))?;
+
+            for (img_index, img) in images.iter().enumerate() {
+                let filters = img.filters.clone().unwrap_or_default();
+
+                all_images.push(PdfImage {
+                    page_number: *page_num as usize,
+                    image_index: img_index + 1,
+                    width: img.width,
+                    height: img.height,
+                    color_space: img.color_space.clone(),
+                    bits_per_component: img.bits_per_component,
+                    filters,
+                    data: img.content.to_vec(),
+                });
+            }
+        }
+
+        Ok(all_images)
+    }
+
+    pub fn extract_images_from_page(&self, page_number: u32) -> Result<Vec<PdfImage>> {
+        let pages = self.document.get_pages();
+        let page_id = pages
+            .get(&page_number)
+            .ok_or(PdfError::PageNotFound(page_number as usize))?;
+
+        let images = self
+            .document
+            .get_page_images(*page_id)
+            .map_err(|e| PdfError::MetadataExtractionFailed(format!("Failed to get page images: {}", e)))?;
+
+        let mut page_images = Vec::new();
+        for (img_index, img) in images.iter().enumerate() {
+            let filters = img.filters.clone().unwrap_or_default();
+
+            page_images.push(PdfImage {
+                page_number: page_number as usize,
+                image_index: img_index + 1,
+                width: img.width,
+                height: img.height,
+                color_space: img.color_space.clone(),
+                bits_per_component: img.bits_per_component,
+                filters,
+                data: img.content.to_vec(),
+            });
+        }
+
+        Ok(page_images)
+    }
+
+    pub fn get_image_count(&self) -> Result<usize> {
+        let images = self.extract_images()?;
+        Ok(images.len())
+    }
+}
+
+pub fn extract_images_from_pdf(pdf_bytes: &[u8]) -> Result<Vec<PdfImage>> {
+    let extractor = PdfImageExtractor::new(pdf_bytes)?;
+    extractor.extract_images()
+}
+
+pub fn extract_images_from_pdf_with_password(pdf_bytes: &[u8], password: &str) -> Result<Vec<PdfImage>> {
+    let extractor = PdfImageExtractor::new_with_password(pdf_bytes, Some(password))?;
+    extractor.extract_images()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_extractor_creation() {
+        let result = PdfImageExtractor::new(b"not a pdf");
+        assert!(result.is_err());
+        assert!(matches!(result.unwrap_err(), PdfError::InvalidPdf(_)));
+    }
+
+    #[test]
+    fn test_extract_images_invalid_pdf() {
+        let result = extract_images_from_pdf(b"not a pdf");
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_extract_images_empty_pdf() {
+        let result = extract_images_from_pdf(b"");
+        assert!(result.is_err());
+    }
+}

data/vendor/kreuzberg/src/pdf/metadata.rs

@@ -0,0 +1,346 @@
+use super::error::{PdfError, Result};
+use pdfium_render::prelude::*;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct PdfMetadata {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub title: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub subject: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub authors: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub keywords: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub created_at: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub modified_at: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub created_by: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub producer: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub page_count: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pdf_version: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub is_encrypted: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub width: Option<i64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub height: Option<i64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub summary: Option<String>,
+}
+
+pub fn extract_metadata(pdf_bytes: &[u8]) -> Result<PdfMetadata> {
+    extract_metadata_with_password(pdf_bytes, None)
+}
+
+pub fn extract_metadata_with_password(pdf_bytes: &[u8], password: Option<&str>) -> Result<PdfMetadata> {
+    let bindings = Pdfium::bind_to_library(Pdfium::pdfium_platform_library_name_at_path("./"))
+        .or_else(|_| Pdfium::bind_to_system_library())
+        .map_err(|e| PdfError::MetadataExtractionFailed(format!("Failed to initialize Pdfium: {}", e)))?;
+
+    let pdfium = Pdfium::new(bindings);
+
+    let document = pdfium.load_pdf_from_byte_slice(pdf_bytes, password).map_err(|e| {
+        let err_msg = e.to_string();
+        if (err_msg.contains("password") || err_msg.contains("Password")) && password.is_some() {
+            PdfError::InvalidPassword
+        } else if err_msg.contains("password") || err_msg.contains("Password") {
+            PdfError::PasswordRequired
+        } else {
+            PdfError::MetadataExtractionFailed(err_msg)
+        }
+    })?;
+
+    extract_metadata_from_document(&document)
+}
+
+pub fn extract_metadata_with_passwords(pdf_bytes: &[u8], passwords: &[&str]) -> Result<PdfMetadata> {
+    let mut last_error = None;
+
+    for password in passwords {
+        match extract_metadata_with_password(pdf_bytes, Some(password)) {
+            Ok(metadata) => return Ok(metadata),
+            Err(err) => {
+                last_error = Some(err);
+                continue;
+            }
+        }
+    }
+
+    if let Some(err) = last_error {
+        return Err(err);
+    }
+
+    extract_metadata(pdf_bytes)
+}
+
+pub(crate) fn extract_metadata_from_document(document: &PdfDocument<'_>) -> Result<PdfMetadata> {
+    let pdf_metadata = document.metadata();
+
+    let mut metadata = PdfMetadata {
+        pdf_version: format_pdf_version(document.version()),
+        ..Default::default()
+    };
+    metadata.page_count = Some(document.pages().len() as usize);
+    metadata.is_encrypted = document
+        .permissions()
+        .security_handler_revision()
+        .ok()
+        .map(|revision| revision != PdfSecurityHandlerRevision::Unprotected);
+
+    metadata.title = pdf_metadata
+        .get(PdfDocumentMetadataTagType::Title)
+        .map(|tag| tag.value().to_string());
+
+    metadata.subject = pdf_metadata
+        .get(PdfDocumentMetadataTagType::Subject)
+        .map(|tag| tag.value().to_string());
+
+    if let Some(author_tag) = pdf_metadata.get(PdfDocumentMetadataTagType::Author) {
+        let authors = parse_authors(author_tag.value());
+        if !authors.is_empty() {
+            metadata.authors = Some(authors);
+        }
+    }
+
+    if let Some(keywords_tag) = pdf_metadata.get(PdfDocumentMetadataTagType::Keywords) {
+        let keywords = parse_keywords(keywords_tag.value());
+        if !keywords.is_empty() {
+            metadata.keywords = Some(keywords);
+        }
+    }
+
+    if let Some(created_tag) = pdf_metadata.get(PdfDocumentMetadataTagType::CreationDate) {
+        metadata.created_at = Some(parse_pdf_date(created_tag.value()));
+    }
+
+    if let Some(modified_tag) = pdf_metadata.get(PdfDocumentMetadataTagType::ModificationDate) {
+        metadata.modified_at = Some(parse_pdf_date(modified_tag.value()));
+    }
+
+    metadata.created_by = pdf_metadata
+        .get(PdfDocumentMetadataTagType::Creator)
+        .map(|tag| tag.value().to_string());
+
+    metadata.producer = pdf_metadata
+        .get(PdfDocumentMetadataTagType::Producer)
+        .map(|tag| tag.value().to_string());
+
+    if !document.pages().is_empty()
+        && let Ok(page_rect) = document.pages().page_size(0)
+    {
+        metadata.width = Some(page_rect.width().value.round() as i64);
+        metadata.height = Some(page_rect.height().value.round() as i64);
+    }
+
+    if metadata.summary.is_none() {
+        metadata.summary = Some(generate_summary(&metadata));
+    }
+
+    Ok(metadata)
+}
+
+fn parse_authors(author_str: &str) -> Vec<String> {
+    let author_str = author_str.replace(" and ", ", ");
+    let mut authors = Vec::new();
+
+    for segment in author_str.split(';') {
+        for author in segment.split(',') {
+            let trimmed = author.trim();
+            if !trimmed.is_empty() {
+                authors.push(trimmed.to_string());
+            }
+        }
+    }
+
+    authors
+}
+
+fn parse_keywords(keywords_str: &str) -> Vec<String> {
+    keywords_str
+        .replace(';', ",")
+        .split(',')
+        .filter_map(|k| {
+            let trimmed = k.trim();
+            if trimmed.is_empty() {
+                None
+            } else {
+                Some(trimmed.to_string())
+            }
+        })
+        .collect()
+}
+
+fn parse_pdf_date(date_str: &str) -> String {
+    let cleaned = date_str.trim();
+
+    if cleaned.starts_with("D:") && cleaned.len() >= 10 {
+        let year = &cleaned[2..6];
+        let month = &cleaned[6..8];
+        let day = &cleaned[8..10];
+
+        if cleaned.len() >= 16 {
+            let hour = &cleaned[10..12];
+            let minute = &cleaned[12..14];
+            let second = &cleaned[14..16];
+            format!("{}-{}-{}T{}:{}:{}Z", year, month, day, hour, minute, second)
+        } else if cleaned.len() >= 14 {
+            let hour = &cleaned[10..12];
+            let minute = &cleaned[12..14];
+            format!("{}-{}-{}T{}:{}:00Z", year, month, day, hour, minute)
+        } else {
+            format!("{}-{}-{}T00:00:00Z", year, month, day)
+        }
+    } else if cleaned.len() >= 8 {
+        let year = &cleaned[0..4];
+        let month = &cleaned[4..6];
+        let day = &cleaned[6..8];
+        format!("{}-{}-{}T00:00:00Z", year, month, day)
+    } else {
+        date_str.to_string()
+    }
+}
+
+fn generate_summary(metadata: &PdfMetadata) -> String {
+    let mut parts = Vec::new();
+
+    if let Some(page_count) = metadata.page_count {
+        let plural = if page_count != 1 { "s" } else { "" };
+        parts.push(format!("PDF document with {} page{}.", page_count, plural));
+    }
+
+    if let Some(ref version) = metadata.pdf_version {
+        parts.push(format!("PDF version {}.", version));
+    }
+
+    if metadata.is_encrypted == Some(true) {
+        parts.push("Document is encrypted.".to_string());
+    }
+
+    parts.join(" ")
+}
+
+fn format_pdf_version(version: PdfDocumentVersion) -> Option<String> {
+    match version {
+        PdfDocumentVersion::Unset => None,
+        PdfDocumentVersion::Pdf1_0 => Some("1.0".to_string()),
+        PdfDocumentVersion::Pdf1_1 => Some("1.1".to_string()),
+        PdfDocumentVersion::Pdf1_2 => Some("1.2".to_string()),
+        PdfDocumentVersion::Pdf1_3 => Some("1.3".to_string()),
+        PdfDocumentVersion::Pdf1_4 => Some("1.4".to_string()),
+        PdfDocumentVersion::Pdf1_5 => Some("1.5".to_string()),
+        PdfDocumentVersion::Pdf1_6 => Some("1.6".to_string()),
+        PdfDocumentVersion::Pdf1_7 => Some("1.7".to_string()),
+        PdfDocumentVersion::Pdf2_0 => Some("2.0".to_string()),
+        PdfDocumentVersion::Other(value) => {
+            if value >= 10 {
+                Some(format!("{}.{}", value / 10, value % 10))
+            } else {
+                Some(value.to_string())
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_authors_single() {
+        let authors = parse_authors("John Doe");
+        assert_eq!(authors, vec!["John Doe"]);
+    }
+
+    #[test]
+    fn test_parse_authors_multiple_comma() {
+        let authors = parse_authors("John Doe, Jane Smith");
+        assert_eq!(authors, vec!["John Doe", "Jane Smith"]);
+    }
+
+    #[test]
+    fn test_parse_authors_multiple_and() {
+        let authors = parse_authors("John Doe and Jane Smith");
+        assert_eq!(authors, vec!["John Doe", "Jane Smith"]);
+    }
+
+    #[test]
+    fn test_parse_authors_semicolon() {
+        let authors = parse_authors("John Doe;Jane Smith");
+        assert_eq!(authors, vec!["John Doe", "Jane Smith"]);
+    }
+
+    #[test]
+    fn test_parse_keywords() {
+        let keywords = parse_keywords("pdf, document, test");
+        assert_eq!(keywords, vec!["pdf", "document", "test"]);
+    }
+
+    #[test]
+    fn test_parse_keywords_semicolon() {
+        let keywords = parse_keywords("pdf;document;test");
+        assert_eq!(keywords, vec!["pdf", "document", "test"]);
+    }
+
+    #[test]
+    fn test_parse_keywords_empty() {
+        let keywords = parse_keywords("");
+        assert!(keywords.is_empty());
+    }
+
+    #[test]
+    fn test_parse_pdf_date_full() {
+        let date = parse_pdf_date("D:20230115123045");
+        assert_eq!(date, "2023-01-15T12:30:45Z");
+    }
+
+    #[test]
+    fn test_parse_pdf_date_no_time() {
+        let date = parse_pdf_date("D:20230115");
+        assert_eq!(date, "2023-01-15T00:00:00Z");
+    }
+
+    #[test]
+    fn test_parse_pdf_date_no_prefix() {
+        let date = parse_pdf_date("20230115");
+        assert_eq!(date, "2023-01-15T00:00:00Z");
+    }
+
+    #[test]
+    fn test_generate_summary() {
+        let metadata = PdfMetadata {
+            page_count: Some(10),
+            pdf_version: Some("1.7".to_string()),
+            is_encrypted: Some(false),
+            ..Default::default()
+        };
+
+        let summary = generate_summary(&metadata);
+        assert!(summary.contains("10 pages"));
+        assert!(summary.contains("1.7"));
+        assert!(!summary.contains("encrypted"));
+    }
+
+    #[test]
+    fn test_generate_summary_single_page() {
+        let metadata = PdfMetadata {
+            page_count: Some(1),
+            ..Default::default()
+        };
+
+        let summary = generate_summary(&metadata);
+        assert!(summary.contains("1 page."));
+        assert!(!summary.contains("pages"));
+    }
+
+    #[test]
+    fn test_extract_metadata_invalid_pdf() {
+        let result = extract_metadata(b"not a pdf");
+        assert!(result.is_err());
+    }
+}

data/vendor/kreuzberg/src/pdf/mod.rs

@@ -0,0 +1,50 @@
+//! PDF document processing utilities.
+//!
+//! This module provides low-level PDF processing functions for text extraction,
+//! metadata parsing, image extraction, and page rendering. Used internally by
+//! the PDF extractor plugin.
+//!
+//! # Features
+//!
+//! - **Text extraction**: Extract text content from PDFs using `pdfium-render`
+//! - **Metadata extraction**: Parse PDF metadata (title, author, creation date, etc.)
+//! - **Image extraction**: Extract embedded images from PDF pages
+//! - **Page rendering**: Render PDF pages to images for OCR processing
+//! - **Error handling**: Comprehensive PDF-specific error types
+//!
+//! # Example
+//!
+//! ```rust
+//! use kreuzberg::pdf::{extract_text_from_pdf, extract_metadata};
+//!
+//! # fn example() -> kreuzberg::Result<()> {
+//! let pdf_bytes = std::fs::read("document.pdf")?;
+//!
+//! // Extract text
+//! let text = extract_text_from_pdf(&pdf_bytes)?;
+//! println!("Text: {}", text);
+//!
+//! // Extract metadata
+//! let metadata = extract_metadata(&pdf_bytes)?;
+//! println!("Page count: {:?}", metadata.page_count);
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! # Note
+//!
+//! This module is always available. The `ocr` feature enables additional
+//! functionality in the PDF extractor for rendering pages to images.
+pub mod error;
+pub mod images;
+pub mod metadata;
+pub mod rendering;
+pub mod table;
+pub mod text;
+
+pub use error::PdfError;
+pub use images::{PdfImage, PdfImageExtractor, extract_images_from_pdf};
+pub use metadata::extract_metadata;
+pub use rendering::{PageRenderOptions, render_page_to_image};
+pub use table::extract_words_from_page;
+pub use text::extract_text_from_pdf;