kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/tests/api_large_pdf_extraction_diagnostics.rs

@@ -1,289 +0,0 @@
-#![cfg(feature = "api")]
-//! Diagnostic tests for large PDF file extraction issues.
-//!
-//! These tests are designed to isolate and identify the root cause of
-//! issues with large PDF file handling in the Kreuzberg API server.
-//!
-//! Current Status:
-//! - 5MB PDF tests are returning HTTP 400 instead of HTTP 200
-//! - This suggests either:
-//!   a) The mock PDF structure is invalid
-//!   b) The PDF extraction logic has issues with the generated content
-//!   c) The multipart parsing is failing on large payloads
-//!
-//! These diagnostic tests help narrow down which component is failing.
-
-use axum::{
-    body::{Body, to_bytes},
-    http::{Request, StatusCode},
-};
-use kreuzberg::{
-    ExtractionConfig,
-    api::{ApiSizeLimits, create_router_with_limits},
-};
-use serde_json::Value;
-use tower::ServiceExt;
-
-/// Test extracting a minimal valid PDF (control test).
-///
-/// This serves as a baseline to verify the API can handle valid PDFs
-/// before testing with large files.
-#[tokio::test]
-async fn test_extract_minimal_valid_pdf() {
-    let router = create_router_with_limits(ExtractionConfig::default(), ApiSizeLimits::from_mb(10, 10));
-
-    let pdf_content = b"%PDF-1.4
-1 0 obj
-<< /Type /Catalog /Pages 2 0 R >>
-endobj
-2 0 obj
-<< /Type /Pages /Kids [3 0 R] /Count 1 >>
-endobj
-3 0 obj
-<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R >>
-endobj
-4 0 obj
-<< >>
-stream
-BT /F1 12 Tf 50 750 Td (Hello) Tj ET
-endstream
-endobj
-xref
-0 5
-0000000000 65535 f
-0000000009 00000 n
-0000000074 00000 n
-0000000133 00000 n
-0000000214 00000 n
-trailer
-<< /Size 5 /Root 1 0 R >>
-startxref
-340
-%%EOF";
-
-    let boundary = "----minimal-pdf";
-    let mut body = Vec::new();
-
-    body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
-    body.extend_from_slice(b"Content-Disposition: form-data; name=\"files\"; filename=\"minimal.pdf\"\r\n");
-    body.extend_from_slice(b"Content-Type: application/pdf\r\n\r\n");
-    body.extend_from_slice(pdf_content);
-    body.extend_from_slice(b"\r\n");
-    body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes());
-
-    let request = Request::builder()
-        .method("POST")
-        .uri("/extract")
-        .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-        .header("content-length", body.len())
-        .body(Body::from(body))
-        .expect("Failed to build request");
-
-    let response = router.oneshot(request).await.expect("Request failed");
-
-    assert_eq!(
-        response.status(),
-        StatusCode::OK,
-        "Minimal PDF should extract successfully. Status: {} indicates baseline is working",
-        response.status()
-    );
-
-    let body = to_bytes(response.into_body(), 1_000_000)
-        .await
-        .expect("Failed to read response body");
-
-    let parsed: Value = serde_json::from_slice(&body).expect("Failed to parse response");
-    eprintln!("Extraction result: {}", serde_json::to_string_pretty(&parsed).unwrap());
-}
-
-/// Test extracting a 1MB text file (control test without PDF).
-///
-/// This isolates whether the issue is specific to PDF handling or
-/// a general problem with large multipart uploads.
-#[tokio::test]
-async fn test_extract_1mb_text_file() {
-    let router = create_router_with_limits(ExtractionConfig::default(), ApiSizeLimits::from_mb(10, 10));
-
-    let boundary = "----large-text";
-    let large_text = "This is test content. ".repeat(50000);
-
-    let mut body = Vec::new();
-    body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
-    body.extend_from_slice(b"Content-Disposition: form-data; name=\"files\"; filename=\"large.txt\"\r\n");
-    body.extend_from_slice(b"Content-Type: text/plain\r\n\r\n");
-    body.extend_from_slice(large_text.as_bytes());
-    body.extend_from_slice(b"\r\n");
-    body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes());
-
-    let request = Request::builder()
-        .method("POST")
-        .uri("/extract")
-        .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-        .header("content-length", body.len())
-        .body(Body::from(body))
-        .expect("Failed to build request");
-
-    let response = router.oneshot(request).await.expect("Request failed");
-
-    println!("1MB text file extraction status: {}", response.status());
-
-    assert_eq!(
-        response.status(),
-        StatusCode::OK,
-        "1MB text file should extract successfully. If this fails, multipart parsing may have issues."
-    );
-}
-
-/// Test extracting progressively larger text files to find breaking point.
-///
-/// This helps identify at what size the API starts failing.
-#[tokio::test]
-async fn test_find_size_breaking_point() {
-    let sizes = vec![
-        ("100KB", 100 * 1024),
-        ("500KB", 500 * 1024),
-        ("1MB", 1024 * 1024),
-        ("2MB", 2 * 1024 * 1024),
-        ("5MB", 5 * 1024 * 1024),
-    ];
-
-    for (label, size) in sizes {
-        let router = create_router_with_limits(ExtractionConfig::default(), ApiSizeLimits::from_mb(20, 20));
-
-        let boundary = "----size-test";
-        let content = "A".repeat(size);
-
-        let mut body = Vec::new();
-        body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
-        body.extend_from_slice(
-            format!(
-                "Content-Disposition: form-data; name=\"files\"; filename=\"test_{}.txt\"\r\n",
-                label
-            )
-            .as_bytes(),
-        );
-        body.extend_from_slice(b"Content-Type: text/plain\r\n\r\n");
-        body.extend_from_slice(content.as_bytes());
-        body.extend_from_slice(b"\r\n");
-        body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes());
-
-        let request = Request::builder()
-            .method("POST")
-            .uri("/extract")
-            .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-            .header("content-length", body.len())
-            .body(Body::from(body))
-            .expect("Failed to build request");
-
-        let response = router.oneshot(request).await.expect("Request failed");
-
-        println!("Size {} ({}B): HTTP {}", label, size, response.status().as_u16());
-
-        if response.status() != StatusCode::OK {
-            eprintln!("Extraction failed at size: {}", label);
-
-            let body = to_bytes(response.into_body(), 1_000_000)
-                .await
-                .expect("Failed to read response body");
-
-            if let Ok(parsed) = serde_json::from_slice::<Value>(&body) {
-                eprintln!("Error response: {}", serde_json::to_string_pretty(&parsed).unwrap());
-            } else {
-                eprintln!("Response body (not JSON): {}", String::from_utf8_lossy(&body));
-            }
-
-            return;
-        }
-    }
-}
-
-/// Test that the default 100MB limit is being applied.
-///
-/// Verifies that the server is actually respecting the configured limits,
-/// and documents what the default limit actually is.
-#[tokio::test]
-async fn test_default_size_limits() {
-    let default_limits = ApiSizeLimits::default();
-    assert_eq!(default_limits.max_request_body_bytes, 100 * 1024 * 1024);
-    assert_eq!(default_limits.max_multipart_field_bytes, 100 * 1024 * 1024);
-
-    println!(
-        "Default limits: {} bytes request, {} bytes per field",
-        default_limits.max_request_body_bytes, default_limits.max_multipart_field_bytes
-    );
-}
-
-/// Test that the router layer actually applies RequestBodyLimitLayer.
-///
-/// Creates a router and verifies that size limit enforcement is active.
-#[tokio::test]
-async fn test_request_body_limit_layer_applied() {
-    let small_limits = ApiSizeLimits::from_mb(1, 1);
-    let router = create_router_with_limits(ExtractionConfig::default(), small_limits);
-
-    let boundary = "----exceed-limits";
-    let large_content = "X".repeat(2 * 1024 * 1024);
-
-    let mut body = Vec::new();
-    body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
-    body.extend_from_slice(b"Content-Disposition: form-data; name=\"files\"; filename=\"test.txt\"\r\n");
-    body.extend_from_slice(b"Content-Type: text/plain\r\n\r\n");
-    body.extend_from_slice(large_content.as_bytes());
-    body.extend_from_slice(b"\r\n");
-    body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes());
-
-    let request = Request::builder()
-        .method("POST")
-        .uri("/extract")
-        .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-        .header("content-length", body.len())
-        .body(Body::from(body))
-        .expect("Failed to build request");
-
-    let response = router.oneshot(request).await.expect("Request failed");
-
-    assert_eq!(
-        response.status(),
-        StatusCode::PAYLOAD_TOO_LARGE,
-        "2MB file should be rejected when limit is 1MB"
-    );
-}
-
-/// Test multipart parsing with incremental content.
-///
-/// Some implementations have issues with streaming multipart parsing.
-/// This test uses proper CRLF line endings to ensure correct parsing.
-#[tokio::test]
-async fn test_multipart_proper_crlf_formatting() {
-    let router = create_router_with_limits(ExtractionConfig::default(), ApiSizeLimits::from_mb(10, 10));
-
-    let content = "Test PDF content that is at least somewhat large for testing purposes.";
-
-    let mut body = Vec::new();
-
-    body.extend_from_slice(b"--BOUNDARY123456\r\n");
-
-    body.extend_from_slice(b"Content-Disposition: form-data; name=\"files\"; filename=\"test.pdf\"\r\n");
-    body.extend_from_slice(b"Content-Type: application/pdf\r\n");
-
-    body.extend_from_slice(b"\r\n");
-
-    body.extend_from_slice(content.as_bytes());
-
-    body.extend_from_slice(b"\r\n");
-
-    body.extend_from_slice(b"--BOUNDARY123456--\r\n");
-
-    let request = Request::builder()
-        .method("POST")
-        .uri("/extract")
-        .header("content-type", "multipart/form-data; boundary=BOUNDARY123456")
-        .header("content-length", body.len())
-        .body(Body::from(body))
-        .expect("Failed to build request");
-
-    let response = router.oneshot(request).await.expect("Request failed");
-
-    println!("Multipart with proper CRLF: HTTP {}", response.status().as_u16());
-    assert!(response.status().is_success() || response.status().is_client_error());
-}

data/vendor/kreuzberg/tests/batch_pooling_benchmark.rs

@@ -1,154 +0,0 @@
-//! Benchmark tests for object pooling in batch extraction.
-//!
-//! This test suite demonstrates the performance benefits of object pooling
-//! during batch document extraction operations.
-
-#[cfg(feature = "tokio-runtime")]
-mod tests {
-    use kreuzberg::core::{BatchProcessor, ExtractionConfig};
-    use kreuzberg::utils::pool::create_string_buffer_pool;
-
-    #[test]
-    fn test_batch_processor_initialization() {
-        let processor = BatchProcessor::new();
-        assert_eq!(processor.string_pool_size(), 0);
-        assert_eq!(processor.byte_pool_size(), 0);
-    }
-
-    #[test]
-    fn test_string_pool_reuse_reduces_allocations() {
-        let pool = create_string_buffer_pool(5, 8192);
-
-        let mut buffers = vec![];
-        for _ in 0..3 {
-            let buf = pool.acquire().unwrap();
-            buffers.push(buf);
-        }
-
-        drop(buffers);
-
-        assert_eq!(pool.size(), 3, "pool should have 3 buffers after first batch");
-
-        let mut buffers = vec![];
-        for _ in 0..3 {
-            let buf = pool.acquire().unwrap();
-            buffers.push(buf);
-        }
-        drop(buffers);
-
-        assert!(pool.size() <= 5, "pool should not exceed max size");
-    }
-
-    #[test]
-    fn test_batch_processor_multiple_operations() {
-        let processor = BatchProcessor::new();
-
-        for _batch in 0..3 {
-            let mut results = vec![];
-
-            for _i in 0..5 {
-                let string_buf = processor.string_pool().acquire().unwrap();
-                let byte_buf = processor.byte_pool().acquire().unwrap();
-
-                results.push((string_buf, byte_buf));
-            }
-
-            drop(results);
-
-            assert!(processor.string_pool_size() <= 10);
-            assert!(processor.byte_pool_size() <= 10);
-        }
-    }
-
-    #[test]
-    fn test_pool_memory_efficiency() {
-        let pool = create_string_buffer_pool(5, 4096);
-
-        let capacity_initial = {
-            let buf = pool.acquire().unwrap();
-            buf.capacity()
-        };
-
-        for _ in 0..10 {
-            let mut buf = pool.acquire().unwrap();
-            buf.push_str("test data");
-        }
-
-        let capacity_final = {
-            let buf = pool.acquire().unwrap();
-            buf.capacity()
-        };
-
-        assert_eq!(
-            capacity_initial, capacity_final,
-            "buffer capacity should be maintained across reuses"
-        );
-    }
-
-    #[tokio::test]
-    async fn test_batch_processor_with_extraction_config() {
-        let processor = BatchProcessor::new();
-        let _config = ExtractionConfig::default();
-
-        assert!(processor.config().string_pool_size > 0);
-        assert!(processor.config().string_buffer_capacity > 0);
-        assert!(processor.config().byte_pool_size > 0);
-        assert!(processor.config().byte_buffer_capacity > 0);
-    }
-
-    #[test]
-    fn test_pool_clear_resets_size() {
-        let processor = BatchProcessor::new();
-
-        {
-            let _s1 = processor.string_pool().acquire().unwrap();
-            let _s2 = processor.string_pool().acquire().unwrap();
-            let _b1 = processor.byte_pool().acquire().unwrap();
-        }
-
-        assert!(processor.string_pool_size() > 0);
-        assert!(processor.byte_pool_size() > 0);
-
-        processor.clear_pools().unwrap();
-
-        assert_eq!(processor.string_pool_size(), 0);
-        assert_eq!(processor.byte_pool_size(), 0);
-    }
-
-    #[test]
-    fn test_concurrent_pool_access() {
-        use std::sync::Arc;
-        use std::thread;
-
-        let processor = Arc::new(BatchProcessor::new());
-        let mut handles = vec![];
-
-        for _thread_id in 0..4 {
-            let processor_clone = Arc::clone(&processor);
-
-            let handle = thread::spawn(move || {
-                for _ in 0..5 {
-                    let _buf1 = processor_clone.string_pool().acquire();
-                    let _buf2 = processor_clone.byte_pool().acquire();
-                }
-            });
-
-            handles.push(handle);
-        }
-
-        for handle in handles {
-            handle.join().unwrap();
-        }
-
-        assert!(processor.string_pool_size() <= 10);
-        assert!(processor.byte_pool_size() <= 10);
-    }
-
-    #[test]
-    fn test_pool_respects_capacity_hints() {
-        let pool = create_string_buffer_pool(3, 2048);
-
-        let buf = pool.acquire().unwrap();
-        assert!(buf.capacity() >= 2048, "buffer should respect capacity hint");
-    }
-}