kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/Cargo.toml

@@ -1,61 +0,0 @@
-[workspace]
-members = ["kreuzberg", "kreuzberg-tesseract", "kreuzberg-ffi"]
-resolver = "2"
-
-[workspace.package]
-version = "4.0.0-rc.29"
-edition = "2024"
-rust-version = "1.91"
-authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
-license = "MIT"
-repository = "https://github.com/kreuzberg-dev/kreuzberg"
-homepage = "https://kreuzberg.dev"
-
-[workspace.dependencies]
-# Core async runtime
-tokio = { version = "1.49.0", features = [
-    "rt",
-    "rt-multi-thread",
-    "macros",
-    "sync",
-    "process",
-    "fs",
-    "time",
-    "io-util",
-] }
-
-# Serialization
-serde = { version = "1.0.228", features = ["derive"] }
-serde_json = "1.0.149"
-
-# Error handling
-thiserror = "2.0.17"
-anyhow = "1.0"
-
-# Async utilities
-async-trait = "0.1.89"
-libc = "0.2.180"
-parking_lot = "0.12.5"
-
-# Tracing/observability
-tracing = "0.1"
-
-# Utilities
-ahash = "0.8.12"
-base64 = "0.22.1"
-hex = "0.4.3"
-toml = "0.9.11"
-num_cpus = "1.17.0"
-once_cell = "1.21.3"
-html-to-markdown-rs = { version = "2.20.0", default-features = false }
-reqwest = { version = "0.13.1", default-features = false, features = ["json", "rustls"] }
-image = { version = "0.25.9", default-features = false }
-lzma-rust2 = { version = "0.15.6" }
-
-# Fix for WASM builds: ensure getrandom has wasm_js feature enabled
-# This is needed because ring/rustls depend on getrandom without the wasm_js feature
-getrandom = { version = "0.3.4", features = ["wasm_js"] }
-
-# Testing (dev)
-tempfile = "3.24.0"
-criterion = { version = "0.8", features = ["html_reports"] }

data/vendor/kreuzberg/examples/bench_fixes.rs

@@ -1,71 +0,0 @@
-use kreuzberg::{ExtractionConfig, extract_file_sync};
-use std::path::PathBuf;
-use std::time::Instant;
-
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let test_pdfs = [
-        (
-            "a_comprehensive_study_of_convergent_and_commutative_replicated_data_types.pdf",
-            "Academic Paper (18 fonts)",
-        ),
-        (
-            "5_level_paging_and_5_level_ept_intel_revision_1_1_may_2017.pdf",
-            "Intel PDF (5 fonts)",
-        ),
-        ("fake_memo.pdf", "Tiny Memo (3-5 fonts)"),
-    ];
-
-    let config = ExtractionConfig {
-        use_cache: false,
-        ..Default::default()
-    };
-
-    println!("=== PDFium Fork Fixes Benchmark ===\n");
-    println!("Testing warm execution fix and font overhead fix\n");
-
-    for (file, description) in &test_pdfs {
-        let path = PathBuf::from(format!("test_documents/pdfs/{}", file));
-        println!("=== {} ===", description);
-        println!("File: {}\n", file);
-
-        let start = Instant::now();
-        let result = extract_file_sync(&path, None, &config)?;
-        let cold = start.elapsed();
-        println!("Cold start:  {:>8.2} ms", cold.as_secs_f64() * 1000.0);
-        println!("Text length: {} chars\n", result.content.len());
-
-        let mut warm_times = Vec::new();
-        for i in 1..=5 {
-            let start = Instant::now();
-            let _ = extract_file_sync(&path, None, &config)?;
-            let warm = start.elapsed();
-            warm_times.push(warm);
-            let speedup = cold.as_micros() as f64 / warm.as_micros() as f64;
-            println!(
-                "Warm {:>2}:     {:>8.2} ms ({:>5.2}x faster than cold)",
-                i,
-                warm.as_secs_f64() * 1000.0,
-                speedup
-            );
-        }
-
-        let avg_warm = warm_times.iter().sum::<std::time::Duration>() / warm_times.len() as u32;
-        let avg_speedup = cold.as_micros() as f64 / avg_warm.as_micros() as f64;
-        println!(
-            "\nAverage warm: {:>8.2} ms ({:>5.2}x faster than cold)",
-            avg_warm.as_secs_f64() * 1000.0,
-            avg_speedup
-        );
-        println!("\n{}\n", "=".repeat(60));
-    }
-
-    println!("\n=== Success Criteria ===");
-    println!("✓ Warm Execution Fix:");
-    println!("  - Warm times should be 1-3x faster than cold (realistic)");
-    println!("  - NOT 100-700x faster (the bug we fixed)");
-    println!("\n✓ Font Overhead Fix:");
-    println!("  - Academic Paper cold: ~130-145ms (matches baseline)");
-    println!("  - NOT 180-195ms (the regression we fixed)");
-
-    Ok(())
-}

data/vendor/kreuzberg/examples/test_pdfium_fork.rs

@@ -1,62 +0,0 @@
-use kreuzberg::{ExtractionConfig, extract_file};
-use std::time::Instant;
-
-#[tokio::main]
-async fn main() {
-    let config = ExtractionConfig {
-        use_cache: false,
-        ..Default::default()
-    };
-
-    println!("Testing PDF extraction with cleaned pdfium-render fork...\n");
-
-    println!("Test 1: fake_memo.pdf");
-    let start = Instant::now();
-    match extract_file("test_documents/pdfs/fake_memo.pdf", None, &config).await {
-        Ok(result) => {
-            let duration = start.elapsed();
-            println!("  ✓ Success! Duration: {:?}", duration);
-            println!("  ✓ Text length: {} chars", result.content.len());
-        }
-        Err(e) => {
-            println!("  ✗ Failed: {}", e);
-            std::process::exit(1);
-        }
-    }
-
-    println!("\nTest 2: Warm iteration");
-    let start = Instant::now();
-    match extract_file("test_documents/pdfs/fake_memo.pdf", None, &config).await {
-        Ok(result) => {
-            let duration = start.elapsed();
-            println!("  ✓ Success! Duration: {:?}", duration);
-            println!("  ✓ Text length: {} chars", result.content.len());
-        }
-        Err(e) => {
-            println!("  ✗ Failed: {}", e);
-            std::process::exit(1);
-        }
-    }
-
-    println!("\nTest 3: Academic Paper (18 fonts)");
-    let start = Instant::now();
-    match extract_file(
-        "test_documents/pdfs/a_comprehensive_study_of_convergent_and_commutative_replicated_data_types.pdf",
-        None,
-        &config,
-    )
-    .await
-    {
-        Ok(result) => {
-            let duration = start.elapsed();
-            println!("  ✓ Success! Duration: {:?}", duration);
-            println!("  ✓ Text length: {} chars", result.content.len());
-        }
-        Err(e) => {
-            println!("  ✗ Failed: {}", e);
-            std::process::exit(1);
-        }
-    }
-
-    println!("\n✅ All tests passed! Cleaned pdfium-render fork is working correctly.");
-}

data/vendor/kreuzberg/src/chunking/processor.rs

@@ -1,219 +0,0 @@
-//! Text chunking post-processor.
-//!
-//! This module provides a PostProcessor plugin that chunks text content in
-//! extraction results.
-
-use crate::plugins::{Plugin, PostProcessor, ProcessingStage};
-use crate::{ExtractionConfig, ExtractionResult, KreuzbergError, Result};
-use async_trait::async_trait;
-
-/// Post-processor that chunks text in document content.
-///
-/// This processor:
-/// - Runs in the Middle processing stage
-/// - Only processes when `config.chunking` is configured
-/// - Stores chunks in `result.chunks`
-/// - Uses configurable chunk size and overlap
-///
-/// # Example
-///
-/// ```rust,no_run
-/// use kreuzberg::plugins::{Plugin, PostProcessor};
-/// use kreuzberg::chunking::processor::ChunkingProcessor;
-///
-/// let processor = ChunkingProcessor;
-/// assert_eq!(processor.name(), "text-chunking");
-/// ```
-#[derive(Debug, Clone, Copy)]
-pub struct ChunkingProcessor;
-
-impl Plugin for ChunkingProcessor {
-    fn name(&self) -> &str {
-        "text-chunking"
-    }
-
-    fn version(&self) -> String {
-        env!("CARGO_PKG_VERSION").to_string()
-    }
-
-    fn initialize(&self) -> Result<()> {
-        Ok(())
-    }
-
-    fn shutdown(&self) -> Result<()> {
-        Ok(())
-    }
-}
-
-#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
-#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
-impl PostProcessor for ChunkingProcessor {
-    async fn process(&self, result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
-        let chunking_config = match &config.chunking {
-            Some(cfg) => cfg,
-            None => return Ok(()),
-        };
-
-        let chunk_config = crate::chunking::ChunkingConfig {
-            max_characters: chunking_config.max_chars,
-            overlap: chunking_config.max_overlap,
-            trim: true,
-            chunker_type: crate::chunking::ChunkerType::Text,
-        };
-
-        let chunking_result = crate::chunking::chunk_text(&result.content, &chunk_config, None)
-            .map_err(|e| KreuzbergError::Other(format!("Chunking failed: {}", e)))?;
-        result.chunks = Some(chunking_result.chunks);
-
-        Ok(())
-    }
-
-    fn processing_stage(&self) -> ProcessingStage {
-        ProcessingStage::Middle
-    }
-
-    fn should_process(&self, _result: &ExtractionResult, config: &ExtractionConfig) -> bool {
-        config.chunking.is_some()
-    }
-
-    fn estimated_duration_ms(&self, result: &ExtractionResult) -> u64 {
-        let text_length = result.content.len();
-        (text_length / 10240).max(1) as u64
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::core::config::ChunkingConfig;
-    use crate::types::Metadata;
-
-    #[tokio::test]
-    async fn test_chunking_processor() {
-        let processor = ChunkingProcessor;
-        let config = ExtractionConfig {
-            chunking: Some(ChunkingConfig {
-                max_chars: 100,
-                max_overlap: 10,
-                embedding: None,
-                preset: None,
-            }),
-            ..Default::default()
-        };
-
-        let mut result = ExtractionResult {
-	            content: "This is a longer text that should be split into multiple chunks to test the chunking processor functionality.".to_string(),
-	            mime_type: "text/plain".to_string(),
-	            metadata: Metadata::default(),
-	            tables: vec![],
-	            detected_languages: None,
-	            chunks: None,
-	            images: None,
-	            pages: None,
-	        };
-
-        processor.process(&mut result, &config).await.unwrap();
-
-        assert!(result.chunks.is_some());
-        let chunks = result.chunks.unwrap();
-        assert!(!chunks.is_empty());
-    }
-
-    #[tokio::test]
-    async fn test_chunking_processor_no_config() {
-        let processor = ChunkingProcessor;
-        let config = ExtractionConfig::default();
-
-        let mut result = ExtractionResult {
-            content: "Some text".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        processor.process(&mut result, &config).await.unwrap();
-
-        assert!(result.chunks.is_none());
-    }
-
-    #[test]
-    fn test_chunking_processor_plugin_interface() {
-        let processor = ChunkingProcessor;
-        assert_eq!(processor.name(), "text-chunking");
-        assert!(!processor.version().is_empty());
-        assert!(processor.initialize().is_ok());
-        assert!(processor.shutdown().is_ok());
-    }
-
-    #[test]
-    fn test_chunking_processor_stage() {
-        let processor = ChunkingProcessor;
-        assert_eq!(processor.processing_stage(), ProcessingStage::Middle);
-    }
-
-    #[test]
-    fn test_chunking_processor_should_process() {
-        let processor = ChunkingProcessor;
-
-        let result = ExtractionResult {
-            content: "Sample text".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        let config_with_chunking = ExtractionConfig {
-            chunking: Some(crate::core::config::ChunkingConfig {
-                max_chars: 100,
-                max_overlap: 10,
-                embedding: None,
-                preset: None,
-            }),
-            ..Default::default()
-        };
-        assert!(processor.should_process(&result, &config_with_chunking));
-
-        let config_without_chunking = ExtractionConfig::default();
-        assert!(!processor.should_process(&result, &config_without_chunking));
-    }
-
-    #[test]
-    fn test_chunking_processor_estimated_duration() {
-        let processor = ChunkingProcessor;
-
-        let short_result = ExtractionResult {
-            content: "Short".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        let long_result = ExtractionResult {
-            content: "a".repeat(100000),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        let short_duration = processor.estimated_duration_ms(&short_result);
-        let long_duration = processor.estimated_duration_ms(&long_result);
-
-        assert!(long_duration > short_duration);
-    }
-}