kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/src/utils/mod.rs

@@ -3,7 +3,6 @@
 //! This module provides:
 //! - Quality processing: clean OCR artifacts, calculate quality scores
 //! - String utilities: safe decoding, mojibake fixing, encoding detection
-//! - Object pooling: reusable pools for batch processing to reduce allocations
 
 #[cfg(feature = "quality")]
 pub mod quality;
@@ -11,21 +10,8 @@ pub mod quality;
 #[cfg(feature = "quality")]
 pub mod string_utils;
 
-pub mod pool;
-pub mod pool_sizing;
-pub mod string_pool;
-
 #[cfg(feature = "quality")]
 pub use quality::{calculate_quality_score, clean_extracted_text, normalize_spaces};
 
 #[cfg(feature = "quality")]
 pub use string_utils::{calculate_text_confidence, fix_mojibake, safe_decode};
-
-pub use pool::{
-    ByteBufferPool, Pool, PoolError, PoolGuard, Recyclable, StringBufferPool, create_byte_buffer_pool,
-    create_string_buffer_pool,
-};
-
-pub use pool_sizing::{PoolSizeHint, estimate_pool_size};
-
-pub use string_pool::{InternedString, intern_language_code, intern_mime_type};

data/vendor/kreuzberg/src/utils/quality.rs

@@ -1,4 +1,3 @@
-use crate::text::utf8_validation;
 use ahash::AHashMap;
 use memchr::{memchr, memchr3};
 use once_cell::sync::Lazy;
@@ -388,11 +387,7 @@ fn clean_repeated_punctuation_ascii(text: &str) -> Option<String> {
         }
     }
 
-    if changed {
-        utf8_validation::string_from_utf8(result).ok()
-    } else {
-        None
-    }
+    if changed { String::from_utf8(result).ok() } else { None }
 }
 
 #[inline]
@@ -466,7 +461,7 @@ pub(crate) fn normalize_whitespace_ascii(text: &str) -> Option<String> {
         }
     }
 
-    let normalized = utf8_validation::string_from_utf8(result).unwrap_or_else(|_| text.to_string());
+    let normalized = String::from_utf8(result).unwrap_or_else(|_| text.to_string());
 
     if changed { Some(normalized) } else { None }
 }
@@ -584,11 +579,7 @@ pub(crate) fn collapse_scattered_ascii(text: &str) -> Option<String> {
         i += 1;
     }
 
-    if changed {
-        utf8_validation::string_from_utf8(result).ok()
-    } else {
-        None
-    }
+    if changed { String::from_utf8(result).ok() } else { None }
 }
 
 /// Collapse redundant whitespace while preserving paragraph boundaries.

data/vendor/kreuzberg/tests/api_tests.rs

@@ -964,509 +964,3 @@ async fn test_extract_default_mime_type() {
             || response.status() == StatusCode::INTERNAL_SERVER_ERROR
     );
 }
-
-/// Test size limits configuration with custom limits.
-#[tokio::test]
-async fn test_size_limits_custom_limits() {
-    use kreuzberg::api::{ApiSizeLimits, create_router_with_limits};
-
-    let config = ExtractionConfig::default();
-    let limits = ApiSizeLimits::from_mb(50, 50);
-    let app = create_router_with_limits(config, limits);
-
-    assert_eq!(limits.max_request_body_bytes, 50 * 1024 * 1024);
-    assert_eq!(limits.max_multipart_field_bytes, 50 * 1024 * 1024);
-
-    let boundary = "----boundary";
-    let file_content = "Test";
-
-    let body_content = format!(
-        "--{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"test.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}--\r\n",
-        boundary, file_content, boundary
-    );
-
-    let response = app
-        .oneshot(
-            Request::builder()
-                .method("POST")
-                .uri("/extract")
-                .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-                .body(Body::from(body_content))
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-
-    assert_eq!(response.status(), StatusCode::OK);
-}
-
-/// Test size limits with asymmetric limits (different request vs field sizes).
-#[tokio::test]
-async fn test_size_limits_asymmetric() {
-    use kreuzberg::api::{ApiSizeLimits, create_router_with_limits};
-
-    let config = ExtractionConfig::default();
-    let limits = ApiSizeLimits::new(100 * 1024 * 1024, 50 * 1024 * 1024);
-    let _app = create_router_with_limits(config, limits);
-
-    assert_eq!(limits.max_request_body_bytes, 100 * 1024 * 1024);
-    assert_eq!(limits.max_multipart_field_bytes, 50 * 1024 * 1024);
-}
-
-/// Test default size limits are 100 MB.
-#[test]
-fn test_default_size_limits_100mb() {
-    use kreuzberg::api::ApiSizeLimits;
-
-    let limits = ApiSizeLimits::default();
-
-    assert_eq!(limits.max_request_body_bytes, 100 * 1024 * 1024);
-    assert_eq!(limits.max_multipart_field_bytes, 100 * 1024 * 1024);
-}
-
-/// Test ApiSizeLimits from_mb convenience method.
-#[test]
-fn test_api_size_limits_from_mb() {
-    use kreuzberg::api::ApiSizeLimits;
-
-    let limits = ApiSizeLimits::from_mb(100, 50);
-    assert_eq!(limits.max_request_body_bytes, 100 * 1024 * 1024);
-    assert_eq!(limits.max_multipart_field_bytes, 50 * 1024 * 1024);
-}
-
-/// Test ApiSizeLimits new method.
-#[test]
-fn test_api_size_limits_new() {
-    use kreuzberg::api::ApiSizeLimits;
-
-    let limits = ApiSizeLimits::new(1_000_000, 500_000);
-    assert_eq!(limits.max_request_body_bytes, 1_000_000);
-    assert_eq!(limits.max_multipart_field_bytes, 500_000);
-}
-
-/// Test extracting a file larger than 2MB (issue #248).
-///
-/// This test verifies that the API can handle files larger than Axum's old
-/// default multipart field limit of 2MB. The issue reported files >2MB being
-/// rejected with HTTP 400, which was due to Axum's default `DefaultBodyLimit`.
-///
-/// With the fix, we now explicitly set `DefaultBodyLimit::max()` to match our
-/// configured size limits (default 10 GB), allowing large file uploads.
-#[tokio::test]
-async fn test_extract_file_larger_than_2mb() {
-    let app = create_router(ExtractionConfig::default());
-
-    let boundary = "----boundary";
-    let file_content = "A".repeat(3 * 1024 * 1024);
-
-    let body_content = format!(
-        "--{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"large.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}--\r\n",
-        boundary, file_content, boundary
-    );
-
-    let response = app
-        .oneshot(
-            Request::builder()
-                .method("POST")
-                .uri("/extract")
-                .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-                .body(Body::from(body_content))
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-
-    assert_eq!(
-        response.status(),
-        StatusCode::OK,
-        "3MB file should be accepted. If this fails with 400 or 413, the size limit fix is not working correctly."
-    );
-
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let results: Vec<serde_json::Value> = serde_json::from_slice(&body).unwrap();
-
-    assert_eq!(results.len(), 1);
-    assert_eq!(results[0]["mime_type"], "text/plain");
-    assert!(results[0]["content"].as_str().unwrap().contains("A"));
-}
-
-/// Test extracting a 2MB file (just above the old Axum limit).
-///
-/// This tests the boundary case at exactly 2MB, which was the old Axum default limit.
-/// Files at this size should now be accepted with the DefaultBodyLimit::max() fix.
-#[tokio::test]
-async fn test_extract_2mb_file() {
-    let app = create_router(ExtractionConfig::default());
-
-    let boundary = "----boundary";
-    let file_content = "X".repeat(2 * 1024 * 1024);
-
-    let body_content = format!(
-        "--{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"boundary.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}--\r\n",
-        boundary, file_content, boundary
-    );
-
-    let response = app
-        .oneshot(
-            Request::builder()
-                .method("POST")
-                .uri("/extract")
-                .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-                .body(Body::from(body_content))
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-
-    assert_eq!(
-        response.status(),
-        StatusCode::OK,
-        "2MB file should be accepted (boundary case)"
-    );
-
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let results: Vec<serde_json::Value> = serde_json::from_slice(&body).unwrap();
-
-    assert_eq!(results.len(), 1);
-    assert_eq!(results[0]["mime_type"], "text/plain");
-    assert!(results[0]["content"].as_str().unwrap().contains("X"));
-}
-
-/// Test extracting a 5MB file.
-///
-/// Verifies that files significantly larger than the old 2MB limit work correctly.
-#[tokio::test]
-async fn test_extract_5mb_file() {
-    let app = create_router(ExtractionConfig::default());
-
-    let boundary = "----boundary";
-    let file_content = "B".repeat(5 * 1024 * 1024);
-
-    let body_content = format!(
-        "--{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"medium.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}--\r\n",
-        boundary, file_content, boundary
-    );
-
-    let response = app
-        .oneshot(
-            Request::builder()
-                .method("POST")
-                .uri("/extract")
-                .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-                .body(Body::from(body_content))
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-
-    assert_eq!(response.status(), StatusCode::OK, "5MB file should be accepted");
-
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let results: Vec<serde_json::Value> = serde_json::from_slice(&body).unwrap();
-
-    assert_eq!(results.len(), 1);
-    assert_eq!(results[0]["mime_type"], "text/plain");
-    assert!(results[0]["content"].as_str().unwrap().contains("B"));
-}
-
-/// Test extracting a 10MB file.
-///
-/// Verifies that moderately large files (10MB) are handled correctly.
-#[tokio::test]
-async fn test_extract_10mb_file() {
-    let app = create_router(ExtractionConfig::default());
-
-    let boundary = "----boundary";
-    let file_content = "C".repeat(10 * 1024 * 1024);
-
-    let body_content = format!(
-        "--{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"large.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}--\r\n",
-        boundary, file_content, boundary
-    );
-
-    let response = app
-        .oneshot(
-            Request::builder()
-                .method("POST")
-                .uri("/extract")
-                .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-                .body(Body::from(body_content))
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-
-    assert_eq!(response.status(), StatusCode::OK, "10MB file should be accepted");
-
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let results: Vec<serde_json::Value> = serde_json::from_slice(&body).unwrap();
-
-    assert_eq!(results.len(), 1);
-    assert_eq!(results[0]["mime_type"], "text/plain");
-    assert!(results[0]["content"].as_str().unwrap().contains("C"));
-}
-
-/// Test extracting a 50MB file (half the default limit).
-///
-/// Verifies that very large files (50MB) are handled correctly,
-/// well within the 100MB default limit.
-#[tokio::test]
-async fn test_extract_50mb_file() {
-    let app = create_router(ExtractionConfig::default());
-
-    let boundary = "----boundary";
-    let file_content = "D".repeat(50 * 1024 * 1024);
-
-    let body_content = format!(
-        "--{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"very_large.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}--\r\n",
-        boundary, file_content, boundary
-    );
-
-    let response = app
-        .oneshot(
-            Request::builder()
-                .method("POST")
-                .uri("/extract")
-                .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-                .body(Body::from(body_content))
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-
-    assert_eq!(response.status(), StatusCode::OK, "50MB file should be accepted");
-
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let results: Vec<serde_json::Value> = serde_json::from_slice(&body).unwrap();
-
-    assert_eq!(results.len(), 1);
-    assert_eq!(results[0]["mime_type"], "text/plain");
-    assert!(results[0]["content"].as_str().unwrap().contains("D"));
-}
-
-/// Test extracting a 90MB file (near the 100MB default limit).
-///
-/// Verifies that very large files close to the default limit (90MB out of 100MB)
-/// are handled correctly. This tests the upper boundary of the default configuration.
-#[tokio::test]
-async fn test_extract_90mb_file() {
-    let app = create_router(ExtractionConfig::default());
-
-    let boundary = "----boundary";
-    let file_content = "E".repeat(90 * 1024 * 1024);
-
-    let body_content = format!(
-        "--{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"huge.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}--\r\n",
-        boundary, file_content, boundary
-    );
-
-    let response = app
-        .oneshot(
-            Request::builder()
-                .method("POST")
-                .uri("/extract")
-                .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-                .body(Body::from(body_content))
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-
-    assert_eq!(
-        response.status(),
-        StatusCode::OK,
-        "90MB file should be accepted (within default 100MB limit)"
-    );
-
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let results: Vec<serde_json::Value> = serde_json::from_slice(&body).unwrap();
-
-    assert_eq!(results.len(), 1);
-    assert_eq!(results[0]["mime_type"], "text/plain");
-    assert!(results[0]["content"].as_str().unwrap().contains("E"));
-}
-
-/// Test extracting a file over the 100MB default limit (HTTP 400/413).
-///
-/// Verifies that files exceeding the 100MB default limit are rejected
-/// with a 4xx error (typically HTTP 400 from tower-http RequestBodyLimitLayer or 413)
-/// rather than silently failing or accepting the request.
-#[tokio::test]
-async fn test_extract_file_over_default_limit() {
-    let app = create_router(ExtractionConfig::default());
-
-    let boundary = "----boundary";
-    let file_content = "F".repeat(101 * 1024 * 1024);
-
-    let body_content = format!(
-        "--{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"over_limit.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}--\r\n",
-        boundary, file_content, boundary
-    );
-
-    let response = app
-        .oneshot(
-            Request::builder()
-                .method("POST")
-                .uri("/extract")
-                .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-                .body(Body::from(body_content))
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-
-    assert!(
-        response.status() == StatusCode::BAD_REQUEST || response.status() == StatusCode::PAYLOAD_TOO_LARGE,
-        "101MB file should be rejected with HTTP 400 or 413 (over 100MB default limit), got {}",
-        response.status()
-    );
-}
-
-/// Test extracting multiple large files within limit.
-///
-/// Verifies that multiple large files can be uploaded together as long as
-/// the total size is within the default 100MB limit.
-#[tokio::test]
-async fn test_extract_multiple_large_files_within_limit() {
-    let app = create_router(ExtractionConfig::default());
-
-    let boundary = "----boundary";
-    let file1_content = "G".repeat(25 * 1024 * 1024);
-    let file2_content = "H".repeat(25 * 1024 * 1024);
-    let file3_content = "I".repeat(25 * 1024 * 1024);
-
-    let body_content = format!(
-        "--{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"file1.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"file2.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"file3.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}--\r\n",
-        boundary, file1_content, boundary, file2_content, boundary, file3_content, boundary
-    );
-
-    let response = app
-        .oneshot(
-            Request::builder()
-                .method("POST")
-                .uri("/extract")
-                .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-                .body(Body::from(body_content))
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-
-    assert_eq!(
-        response.status(),
-        StatusCode::OK,
-        "Multiple files totaling 75MB should be accepted"
-    );
-
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let results: Vec<serde_json::Value> = serde_json::from_slice(&body).unwrap();
-
-    assert_eq!(results.len(), 3, "Should have 3 results");
-    for result in &results {
-        assert_eq!(result["mime_type"], "text/plain");
-    }
-    assert!(results[0]["content"].as_str().unwrap().contains("G"));
-    assert!(results[1]["content"].as_str().unwrap().contains("H"));
-    assert!(results[2]["content"].as_str().unwrap().contains("I"));
-}
-
-/// Test extracting multiple large files exceeding limit (HTTP 400/413).
-///
-/// Verifies that when multiple files together exceed the 100MB default limit,
-/// the request is rejected with a 4xx error (typically HTTP 400 or 413).
-#[tokio::test]
-async fn test_extract_multiple_large_files_exceeding_limit() {
-    let app = create_router(ExtractionConfig::default());
-
-    let boundary = "----boundary";
-    let file1_content = "J".repeat(50 * 1024 * 1024);
-    let file2_content = "K".repeat(55 * 1024 * 1024);
-
-    let body_content = format!(
-        "--{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"file1.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}\r\n\
-         Content-Disposition: form-data; name=\"files\"; filename=\"file2.txt\"\r\n\
-         Content-Type: text/plain\r\n\
-         \r\n\
-         {}\r\n\
-         --{}--\r\n",
-        boundary, file1_content, boundary, file2_content, boundary
-    );
-
-    let response = app
-        .oneshot(
-            Request::builder()
-                .method("POST")
-                .uri("/extract")
-                .header("content-type", format!("multipart/form-data; boundary={}", boundary))
-                .body(Body::from(body_content))
-                .unwrap(),
-        )
-        .await
-        .unwrap();
-
-    assert!(
-        response.status() == StatusCode::BAD_REQUEST || response.status() == StatusCode::PAYLOAD_TOO_LARGE,
-        "Multiple files totaling 105MB should be rejected with HTTP 400 or 413, got {}",
-        response.status()
-    );
-}

data/vendor/kreuzberg/tests/archive_integration.rs

@@ -3,8 +3,6 @@
 //! Tests for ZIP, TAR, TAR.GZ, and 7z archive extraction.
 //! Validates metadata extraction, content extraction, nested archives, and error handling.
 
-#![cfg(feature = "archives")]
-
 use kreuzberg::core::config::ExtractionConfig;
 use kreuzberg::core::extractor::{extract_bytes, extract_bytes_sync};
 use std::io::{Cursor, Write};