kreuzberg 4.0.8 → 4.1.1

data/vendor/kreuzberg/tests/api_chunk.rs

@@ -0,0 +1,313 @@
+//! Integration tests for the /chunk API endpoint.
+
+#![cfg(feature = "api")]
+
+use axum::{
+    body::Body,
+    http::{Request, StatusCode},
+};
+use serde_json::json;
+use tower::ServiceExt;
+
+use kreuzberg::{ExtractionConfig, api::create_router};
+
+#[tokio::test]
+async fn test_chunk_basic() {
+    let app = create_router(ExtractionConfig::default());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/chunk")
+                .method("POST")
+                .header("content-type", "application/json")
+                .body(Body::from(
+                    json!({
+                        "text": "Short text. More text here. Even more content to chunk."
+                    })
+                    .to_string(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+}
+
+#[tokio::test]
+async fn test_chunk_empty_text_returns_400() {
+    let app = create_router(ExtractionConfig::default());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/chunk")
+                .method("POST")
+                .header("content-type", "application/json")
+                .body(Body::from(json!({"text": ""}).to_string()))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+}
+
+#[tokio::test]
+async fn test_chunk_markdown_strategy() {
+    let app = create_router(ExtractionConfig::default());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/chunk")
+                .method("POST")
+                .header("content-type", "application/json")
+                .body(Body::from(
+                    json!({
+                        "text": "# Heading\n\nParagraph text here.",
+                        "chunker_type": "markdown"
+                    })
+                    .to_string(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+}
+
+#[tokio::test]
+async fn test_chunk_response_structure() {
+    use kreuzberg::api::ChunkResponse;
+
+    let app = create_router(ExtractionConfig::default());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/chunk")
+                .method("POST")
+                .header("content-type", "application/json")
+                .body(Body::from(
+                    json!({
+                        "text": "This is a test. Another sentence here. And one more sentence to ensure we get chunks.",
+                        "config": {
+                            "max_characters": 50,
+                            "overlap": 10,
+                            "trim": true
+                        },
+                        "chunker_type": "text"
+                    })
+                    .to_string(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+
+    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
+    let chunk_response: ChunkResponse = serde_json::from_slice(&body).unwrap();
+
+    // Verify response structure
+    assert!(chunk_response.chunk_count > 0);
+    assert_eq!(chunk_response.chunks.len(), chunk_response.chunk_count);
+    assert_eq!(chunk_response.chunker_type, "text");
+    assert_eq!(chunk_response.config.max_characters, 50);
+    assert_eq!(chunk_response.config.overlap, 10);
+    assert!(chunk_response.config.trim);
+    assert!(chunk_response.input_size_bytes > 0);
+
+    // Verify chunk metadata
+    for (idx, chunk) in chunk_response.chunks.iter().enumerate() {
+        assert!(!chunk.content.is_empty());
+        assert_eq!(chunk.chunk_index, idx);
+        assert_eq!(chunk.total_chunks, chunk_response.chunk_count);
+        assert!(chunk.byte_end > chunk.byte_start);
+    }
+}
+
+#[tokio::test]
+async fn test_chunk_invalid_strategy_returns_400() {
+    let app = create_router(ExtractionConfig::default());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/chunk")
+                .method("POST")
+                .header("content-type", "application/json")
+                .body(Body::from(
+                    json!({
+                        "text": "Test text",
+                        "chunker_type": "invalid_type"
+                    })
+                    .to_string(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+}
+
+#[tokio::test]
+async fn test_chunk_with_defaults() {
+    use kreuzberg::api::ChunkResponse;
+
+    let app = create_router(ExtractionConfig::default());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/chunk")
+                .method("POST")
+                .header("content-type", "application/json")
+                .body(Body::from(
+                    json!({
+                        "text": "This is a test sentence. Another sentence here."
+                    })
+                    .to_string(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+
+    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
+    let chunk_response: ChunkResponse = serde_json::from_slice(&body).unwrap();
+
+    // Verify defaults are applied
+    assert_eq!(chunk_response.config.max_characters, 2000);
+    assert_eq!(chunk_response.config.overlap, 100);
+    assert!(chunk_response.config.trim);
+    assert_eq!(chunk_response.chunker_type, "text");
+}
+
+#[tokio::test]
+async fn test_chunk_malformed_json_returns_400() {
+    let app = create_router(ExtractionConfig::default());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/chunk")
+                .method("POST")
+                .header("content-type", "application/json")
+                .body(Body::from("{invalid json}"))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+}
+
+#[tokio::test]
+async fn test_chunk_case_insensitive_chunker_type() {
+    use kreuzberg::api::ChunkResponse;
+
+    let app = create_router(ExtractionConfig::default());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/chunk")
+                .method("POST")
+                .header("content-type", "application/json")
+                .body(Body::from(
+                    json!({
+                        "text": "# Title\n\nContent here.",
+                        "chunker_type": "MARKDOWN"
+                    })
+                    .to_string(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+
+    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
+    let chunk_response: ChunkResponse = serde_json::from_slice(&body).unwrap();
+
+    // Verify it's normalized to lowercase
+    assert_eq!(chunk_response.chunker_type, "markdown");
+}
+
+#[tokio::test]
+async fn test_chunk_long_text() {
+    use kreuzberg::api::ChunkResponse;
+
+    let app = create_router(ExtractionConfig::default());
+    let long_text = "Lorem ipsum dolor sit amet. ".repeat(200);
+
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/chunk")
+                .method("POST")
+                .header("content-type", "application/json")
+                .body(Body::from(
+                    json!({
+                        "text": long_text,
+                        "config": {
+                            "max_characters": 500,
+                            "overlap": 50
+                        }
+                    })
+                    .to_string(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+
+    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
+    let chunk_response: ChunkResponse = serde_json::from_slice(&body).unwrap();
+
+    // Should have multiple chunks
+    assert!(chunk_response.chunk_count > 1);
+    assert_eq!(chunk_response.chunks.len(), chunk_response.chunk_count);
+}
+
+#[tokio::test]
+async fn test_chunk_custom_config() {
+    use kreuzberg::api::ChunkResponse;
+
+    let app = create_router(ExtractionConfig::default());
+    let response = app
+        .oneshot(
+            Request::builder()
+                .uri("/chunk")
+                .method("POST")
+                .header("content-type", "application/json")
+                .body(Body::from(
+                    json!({
+                        "text": "Test sentence one. Test sentence two. Test sentence three.",
+                        "config": {
+                            "max_characters": 30,
+                            "overlap": 5,
+                            "trim": false
+                        },
+                        "chunker_type": "text"
+                    })
+                    .to_string(),
+                ))
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+
+    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
+    let chunk_response: ChunkResponse = serde_json::from_slice(&body).unwrap();
+
+    // Verify custom config was applied
+    assert_eq!(chunk_response.config.max_characters, 30);
+    assert_eq!(chunk_response.config.overlap, 5);
+    assert!(!chunk_response.config.trim);
+}

data/vendor/kreuzberg/tests/api_embed.rs

@@ -84,9 +84,8 @@ async fn test_embed_with_custom_config() {
         "texts": ["Test embedding with custom config"],
         "config": {
             "model": {
-                "preset": {
-                    "name": "fast"
-                }
+                "type": "preset",
+                "name": "fast"
             },
             "batch_size": 32
         }
@@ -295,9 +294,8 @@ async fn test_embed_different_presets() {
         "texts": ["Test text"],
         "config": {
             "model": {
-                "preset": {
-                    "name": "fast"
-                }
+                "type": "preset",
+                "name": "fast"
             }
         }
     });
@@ -327,9 +325,8 @@ async fn test_embed_different_presets() {
         "texts": ["Test text"],
         "config": {
             "model": {
-                "preset": {
-                    "name": "balanced"
-                }
+                "type": "preset",
+                "name": "balanced"
             }
         }
     });

data/vendor/kreuzberg/tests/batch_orchestration.rs

@@ -257,6 +257,7 @@ fn test_ocr_multipage_efficiency() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         use_cache: true,

data/vendor/kreuzberg/tests/concurrency_stress.rs

@@ -193,6 +193,7 @@ async fn test_concurrent_ocr_processing() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         use_cache: true,
@@ -262,6 +263,7 @@ fn test_concurrent_ocr_cache_stress() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         use_cache: true,
@@ -313,7 +315,10 @@ fn test_concurrent_ocr_cache_stress() {
 /// - Pipeline can process multiple results in parallel
 /// - Processors don't interfere with each other
 /// - Registry reads are thread-safe
+///
+/// Note: This test is flaky due to timing-dependent concurrent operations.
 #[tokio::test]
+#[ignore = "flaky concurrency test - timing dependent on system load"]
 async fn test_concurrent_pipeline_processing() {
     struct ConcurrentTestProcessor;
 
@@ -378,6 +383,8 @@ async fn test_concurrent_pipeline_processing() {
                 chunks: None,
                 images: None,
                 pages: None,
+                elements: None,
+                djot_content: None,
             };
 
             run_pipeline(result, &config).await

data/vendor/kreuzberg/tests/core_integration.rs

@@ -459,6 +459,7 @@ async fn test_extraction_with_ocr_config() {
             tesseract_config: None,
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()

data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs

@@ -120,3 +120,133 @@ async fn test_docx_minimal_metadata_extraction() {
 
     println!("✅ DOCX minimal metadata extraction test passed!");
 }
+
+#[tokio::test]
+async fn test_docx_keywords_extraction() {
+    // This test verifies that DOCX keywords metadata is properly parsed
+    // from comma-separated strings into Vec<String> in Metadata.keywords
+    //
+    // Addresses GitHub issue #309: DOCX keyword extraction was returning
+    // strings instead of parsed keyword lists, causing FunctionClauseError
+    // in the Elixir binding.
+
+    use std::io::Write;
+    use tempfile::NamedTempFile;
+    use zip::CompressionMethod;
+    use zip::write::{FileOptions, ZipWriter};
+
+    // Create a minimal DOCX with keywords metadata
+    let mut temp_file = NamedTempFile::new().expect("Failed to create temp file");
+
+    {
+        let mut zip = ZipWriter::new(&mut temp_file);
+        let options: FileOptions<()> = FileOptions::default().compression_method(CompressionMethod::Stored);
+
+        // Add [Content_Types].xml
+        zip.start_file("[Content_Types].xml", options).unwrap();
+        zip.write_all(br#"<?xml version="1.0" encoding="UTF-8"?>
+<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
+  <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
+  <Default Extension="xml" ContentType="application/xml"/>
+  <Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
+  <Override PartName="/docProps/core.xml" ContentType="application/vnd.openxmlformats-package.core-properties+xml"/>
+</Types>"#).unwrap();
+
+        // Add _rels/.rels
+        zip.start_file("_rels/.rels", options).unwrap();
+        zip.write_all(br#"<?xml version="1.0" encoding="UTF-8"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+  <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
+  <Relationship Id="rId2" Type="http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties" Target="docProps/core.xml"/>
+</Relationships>"#).unwrap();
+
+        // Add word/document.xml with simple content
+        zip.start_file("word/document.xml", options).unwrap();
+        zip.write_all(
+            br#"<?xml version="1.0" encoding="UTF-8"?>
+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
+  <w:body>
+    <w:p>
+      <w:r>
+        <w:t>Test document for keyword extraction</w:t>
+      </w:r>
+    </w:p>
+  </w:body>
+</w:document>"#,
+        )
+        .unwrap();
+
+        // Add docProps/core.xml with keywords (comma-separated string)
+        zip.start_file("docProps/core.xml", options).unwrap();
+        zip.write_all(
+            br#"<?xml version="1.0" encoding="UTF-8"?>
+<cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/metadata/core-properties"
+                   xmlns:dc="http://purl.org/dc/elements/1.1/"
+                   xmlns:dcterms="http://purl.org/dc/terms/">
+  <dc:title>Test Document</dc:title>
+  <dc:creator>Test Author</dc:creator>
+  <cp:keywords>rust, docx, extraction, metadata, test</cp:keywords>
+  <dc:subject>Testing keyword extraction</dc:subject>
+</cp:coreProperties>"#,
+        )
+        .unwrap();
+
+        zip.finish().unwrap();
+    }
+
+    // Extract the DOCX file
+    let result = extract_file(
+        temp_file.path(),
+        Some("application/vnd.openxmlformats-officedocument.wordprocessingml.document"),
+        &ExtractionConfig::default(),
+    )
+    .await
+    .expect("Should extract DOCX with keywords successfully");
+
+    // Verify content was extracted
+    assert!(!result.content.is_empty(), "Content should not be empty");
+    assert!(
+        result.content.contains("Test document for keyword extraction"),
+        "Content should match document text"
+    );
+
+    // Verify keywords were parsed into Vec<String> in Metadata.keywords
+    assert!(
+        result.metadata.keywords.is_some(),
+        "Keywords should be present in metadata.keywords"
+    );
+
+    let keywords = result.metadata.keywords.as_ref().unwrap();
+    assert_eq!(
+        keywords.len(),
+        5,
+        "Should have 5 keywords parsed from comma-separated string"
+    );
+
+    // Verify individual keywords were trimmed and parsed correctly
+    assert_eq!(keywords[0], "rust", "First keyword should be 'rust'");
+    assert_eq!(keywords[1], "docx", "Second keyword should be 'docx'");
+    assert_eq!(keywords[2], "extraction", "Third keyword should be 'extraction'");
+    assert_eq!(keywords[3], "metadata", "Fourth keyword should be 'metadata'");
+    assert_eq!(keywords[4], "test", "Fifth keyword should be 'test'");
+
+    // Verify other metadata was also extracted
+    assert_eq!(
+        result.metadata.additional.get("created_by").and_then(|v| v.as_str()),
+        Some("Test Author"),
+        "Should have correct creator"
+    );
+    assert_eq!(
+        result.metadata.additional.get("title").and_then(|v| v.as_str()),
+        Some("Test Document"),
+        "Should have correct title"
+    );
+    assert_eq!(
+        result.metadata.additional.get("subject").and_then(|v| v.as_str()),
+        Some("Testing keyword extraction"),
+        "Should have correct subject"
+    );
+
+    println!("✅ DOCX keywords extraction test passed!");
+    println!("   Extracted keywords: {:?}", keywords);
+}

data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs

@@ -53,20 +53,14 @@ async fn test_native_epub_wasteland_extraction() {
         result.content.len()
     );
 
-    assert!(
-        result.metadata.additional.contains_key("title"),
-        "Should extract title metadata"
-    );
+    assert!(result.metadata.title.is_some(), "Should extract title metadata");
     assert_eq!(
-        result.metadata.additional.get("title").and_then(|v| v.as_str()),
+        result.metadata.title.as_deref(),
         Some("The Waste Land"),
         "Should have correct title"
     );
 
-    assert!(
-        result.metadata.additional.contains_key("creator"),
-        "Should extract creator metadata"
-    );
+    assert!(result.metadata.authors.is_some(), "Should extract creator metadata");
 
     assert!(
         result.content.contains("April") || result.content.contains("cruellest"),
@@ -105,10 +99,7 @@ async fn test_native_epub_images_extraction() {
         result.content.len()
     );
 
-    assert!(
-        result.metadata.additional.contains_key("title"),
-        "Should extract title metadata"
-    );
+    assert!(result.metadata.title.is_some(), "Should extract title metadata");
 
     println!("✅ Images EPUB extraction test passed ({} bytes)", result.content.len());
 }
@@ -179,7 +170,7 @@ async fn test_native_epub2_cover_extraction() {
     );
 
     assert_eq!(
-        result.metadata.additional.get("title").and_then(|v| v.as_str()),
+        result.metadata.title.as_deref(),
         Some("Pandoc EPUB Test"),
         "Should have correct title"
     );

data/vendor/kreuzberg/tests/format_integration.rs

@@ -100,6 +100,7 @@ async fn test_ocr_simple_english_image_async() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -142,6 +143,7 @@ async fn test_ocr_image_without_text_async() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()

data/vendor/kreuzberg/tests/helpers/mod.rs

@@ -115,6 +115,7 @@ pub fn test_config_with_ocr() -> kreuzberg::core::config::ExtractionConfig {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()