kreuzberg 4.1.2 → 4.2.1

data/vendor/kreuzberg/tests/rtf_extractor_tests.rs

@@ -74,7 +74,7 @@ async fn test_rtf_accent_extraction() {
     let result = extract_file(&path, Some("application/rtf"), &config).await;
 
     assert!(result.is_ok(), "RTF extraction should succeed for accent.rtf");
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     assert_eq!(extraction.mime_type, "application/rtf");
 
@@ -112,7 +112,7 @@ async fn test_rtf_bookmark_extraction() {
     let result = extract_file(&path, Some("application/rtf"), &config).await;
 
     assert!(result.is_ok(), "RTF extraction should succeed for bookmark.rtf");
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     let content = extraction.content.to_lowercase();
 
@@ -137,7 +137,7 @@ async fn test_rtf_footnote_extraction() {
     let result = extract_file(&path, Some("application/rtf"), &config).await;
 
     assert!(result.is_ok(), "RTF extraction should succeed for footnote.rtf");
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     assert!(!extraction.content.is_empty(), "Content should not be empty");
 
@@ -176,7 +176,7 @@ async fn test_rtf_formatting_extraction() {
     let result = extract_file(&path, Some("application/rtf"), &config).await;
 
     assert!(result.is_ok(), "RTF extraction should succeed for formatting.rtf");
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     assert!(!extraction.content.is_empty(), "Content should not be empty");
 
@@ -223,7 +223,7 @@ async fn test_rtf_heading_extraction() {
     let result = extract_file(&path, Some("application/rtf"), &config).await;
 
     assert!(result.is_ok(), "RTF extraction should succeed for heading.rtf");
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     assert!(!extraction.content.is_empty(), "Content should not be empty");
 
@@ -269,7 +269,7 @@ async fn test_rtf_image_extraction() {
     let result = extract_file(&path, Some("application/rtf"), &config).await;
 
     assert!(result.is_ok(), "RTF extraction should succeed for image.rtf");
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     assert!(!extraction.content.is_empty(), "Content should not be empty");
 
@@ -301,7 +301,7 @@ async fn test_rtf_link_extraction() {
     let result = extract_file(&path, Some("application/rtf"), &config).await;
 
     assert!(result.is_ok(), "RTF extraction should succeed for link.rtf");
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     assert!(!extraction.content.is_empty(), "Content should not be empty");
 
@@ -328,7 +328,7 @@ async fn test_rtf_list_complex_extraction() {
     let result = extract_file(&path, Some("application/rtf"), &config).await;
 
     assert!(result.is_ok(), "RTF extraction should succeed for list_complex.rtf");
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     assert!(!extraction.content.is_empty(), "Content should not be empty");
 
@@ -381,7 +381,7 @@ async fn test_rtf_list_simple_extraction() {
     let result = extract_file(&path, Some("application/rtf"), &config).await;
 
     assert!(result.is_ok(), "RTF extraction should succeed for list_simple.rtf");
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     assert!(!extraction.content.is_empty(), "Content should not be empty");
 
@@ -422,7 +422,7 @@ async fn test_rtf_table_error_codes_extraction() {
         result.is_ok(),
         "RTF extraction should succeed for table_error_codes.rtf"
     );
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     assert!(
         extraction.mime_type == "application/rtf",
@@ -448,7 +448,7 @@ async fn test_rtf_table_simple_extraction() {
     let result = extract_file(&path, Some("application/rtf"), &config).await;
 
     assert!(result.is_ok(), "RTF extraction should succeed for table_simple.rtf");
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     assert!(
         extraction.mime_type == "application/rtf",
@@ -470,7 +470,7 @@ async fn test_rtf_unicode_extraction() {
     let result = extract_file(&path, Some("application/rtf"), &config).await;
 
     assert!(result.is_ok(), "RTF extraction should succeed for unicode.rtf");
-    let extraction = result.unwrap();
+    let extraction = result.expect("Operation failed");
 
     assert!(!extraction.content.is_empty(), "Content should not be empty");
 
@@ -493,8 +493,8 @@ async fn test_rtf_extraction_deterministic_unicode() {
 
     assert!(result1.is_ok() && result2.is_ok(), "Both extractions should succeed");
 
-    let extraction1 = result1.unwrap();
-    let extraction2 = result2.unwrap();
+    let extraction1 = result1.expect("Operation failed");
+    let extraction2 = result2.expect("Operation failed");
 
     assert_eq!(
         extraction1.content, extraction2.content,
@@ -514,8 +514,8 @@ async fn test_rtf_extraction_deterministic_list_complex() {
 
     assert!(result1.is_ok() && result2.is_ok(), "Both extractions should succeed");
 
-    let extraction1 = result1.unwrap();
-    let extraction2 = result2.unwrap();
+    let extraction1 = result1.expect("Operation failed");
+    let extraction2 = result2.expect("Operation failed");
 
     assert_eq!(
         extraction1.content, extraction2.content,
@@ -551,7 +551,7 @@ async fn test_rtf_no_critical_content_loss() {
             filename
         );
 
-        let extraction = result.unwrap();
+        let extraction = result.expect("Operation failed");
         assert!(
             !extraction.content.is_empty(),
             "FAIL: CRITICAL - Extracted 0 bytes from {}. RTF extractor lost all content.",
@@ -582,7 +582,7 @@ async fn test_rtf_mime_type_preservation() {
 
         assert!(result.is_ok(), "Extraction should succeed for {}", filename);
 
-        let extraction = result.unwrap();
+        let extraction = result.expect("Operation failed");
         assert_eq!(
             extraction.mime_type, "application/rtf",
             "FAIL: MIME type not preserved for {}",

data/vendor/kreuzberg/tests/security_validation.rs

@@ -31,11 +31,11 @@ fn test_archive_zip_bomb_detection() {
         let mut zip = ZipWriter::new(&mut cursor);
         let options = FileOptions::<'_, ()>::default();
 
-        zip.start_file("large.txt", options).unwrap();
+        zip.start_file("large.txt", options).expect("Operation failed");
         let zeros = vec![0u8; 10 * 1024 * 1024];
-        zip.write_all(&zeros).unwrap();
+        zip.write_all(&zeros).expect("Operation failed");
 
-        zip.finish().unwrap();
+        zip.finish().expect("Operation failed");
     }
 
     let bytes = cursor.into_inner();
@@ -57,10 +57,10 @@ fn test_archive_path_traversal_zip() {
         let mut zip = ZipWriter::new(&mut cursor);
         let options = FileOptions::<'_, ()>::default();
 
-        zip.start_file("../../etc/passwd", options).unwrap();
-        zip.write_all(b"malicious content").unwrap();
+        zip.start_file("../../etc/passwd", options).expect("Operation failed");
+        zip.write_all(b"malicious content").expect("Operation failed");
 
-        zip.finish().unwrap();
+        zip.finish().expect("Operation failed");
     }
 
     let bytes = cursor.into_inner();
@@ -97,10 +97,10 @@ fn test_archive_absolute_paths_rejected() {
         let mut zip = ZipWriter::new(&mut cursor);
         let options = FileOptions::<'_, ()>::default();
 
-        zip.start_file("/tmp/malicious.txt", options).unwrap();
-        zip.write_all(b"malicious content").unwrap();
+        zip.start_file("/tmp/malicious.txt", options).expect("Operation failed");
+        zip.write_all(b"malicious content").expect("Operation failed");
 
-        zip.finish().unwrap();
+        zip.finish().expect("Operation failed");
     }
 
     let bytes = cursor.into_inner();
@@ -125,10 +125,10 @@ fn test_archive_deeply_nested_directories() {
         let deep_path = (0..100).map(|i| format!("dir{}", i)).collect::<Vec<_>>().join("/");
         let file_path = format!("{}/file.txt", deep_path);
 
-        zip.start_file(&file_path, options).unwrap();
-        zip.write_all(b"deep content").unwrap();
+        zip.start_file(&file_path, options).expect("Operation failed");
+        zip.write_all(b"deep content").expect("Operation failed");
 
-        zip.finish().unwrap();
+        zip.finish().expect("Operation failed");
     }
 
     let bytes = cursor.into_inner();
@@ -149,11 +149,12 @@ fn test_archive_many_small_files() {
         let options = FileOptions::<'_, ()>::default();
 
         for i in 0..1000 {
-            zip.start_file(format!("file{}.txt", i), options).unwrap();
-            zip.write_all(b"small content").unwrap();
+            zip.start_file(format!("file{}.txt", i), options)
+                .expect("Operation failed");
+            zip.write_all(b"small content").expect("Operation failed");
         }
 
-        zip.finish().unwrap();
+        zip.finish().expect("Operation failed");
     }
 
     let bytes = cursor.into_inner();
@@ -404,13 +405,13 @@ fn test_security_directory_instead_of_file() {
 
 #[test]
 fn test_security_special_file_handling() {
-    let mut tmpfile = NamedTempFile::new().unwrap();
-    tmpfile.write_all(b"test content").unwrap();
-    tmpfile.flush().unwrap();
+    let mut tmpfile = NamedTempFile::new().expect("Operation failed");
+    tmpfile.write_all(b"test content").expect("Operation failed");
+    tmpfile.flush().expect("Operation failed");
     let path = tmpfile.path();
 
     let config = ExtractionConfig::default();
-    let result = extract_file_sync(path.to_str().unwrap(), None, &config);
+    let result = extract_file_sync(path.to_str().expect("Operation failed"), None, &config);
 
     assert!(result.is_ok() || result.is_err());
 }

data/vendor/kreuzberg/tests/serialization_integration.rs

@@ -0,0 +1,112 @@
+//! Cross-language serialization integration tests.
+//!
+//! These tests validate that ExtractionConfig serializes correctly
+//! and that the serialized output can be used for cross-language comparison.
+
+use kreuzberg::core::config::ExtractionConfig;
+
+#[test]
+fn test_extraction_config_minimal_serialization() {
+    let config = ExtractionConfig::default();
+    let json = serde_json::to_value(&config).expect("Failed to serialize config");
+
+    // Validate that all expected fields are present
+    assert!(json.get("use_cache").is_some(), "Missing use_cache field");
+    assert!(
+        json.get("enable_quality_processing").is_some(),
+        "Missing enable_quality_processing field"
+    );
+    assert!(json.get("force_ocr").is_some(), "Missing force_ocr field");
+}
+
+#[test]
+fn test_extraction_config_serialization_round_trip() {
+    let original = ExtractionConfig {
+        use_cache: true,
+        enable_quality_processing: false,
+        force_ocr: true,
+        ..Default::default()
+    };
+
+    // Serialize to JSON
+    let json = serde_json::to_value(&original).expect("Failed to serialize");
+
+    // Deserialize back
+    let restored: ExtractionConfig = serde_json::from_value(json).expect("Failed to deserialize");
+
+    // Validate that key fields are preserved
+    assert_eq!(original.use_cache, restored.use_cache, "use_cache field not preserved");
+    assert_eq!(
+        original.enable_quality_processing, restored.enable_quality_processing,
+        "enable_quality_processing field not preserved"
+    );
+    assert_eq!(original.force_ocr, restored.force_ocr, "force_ocr field not preserved");
+}
+
+#[test]
+fn test_extraction_config_nested_serialization() {
+    let config = ExtractionConfig {
+        use_cache: true,
+        enable_quality_processing: true,
+        force_ocr: false,
+        // Note: Nested fields like ocr, chunking, etc. would be set here
+        // This test focuses on the basic serialization structure
+        ..Default::default()
+    };
+
+    let json = serde_json::to_value(&config).expect("Failed to serialize");
+
+    // Ensure it's a proper JSON object
+    assert!(json.is_object(), "Serialized output should be a JSON object");
+
+    // Validate that core fields are present
+    assert!(json.get("use_cache").is_some());
+    assert!(json.get("enable_quality_processing").is_some());
+    assert!(json.get("force_ocr").is_some());
+}
+
+#[test]
+fn test_extraction_config_json_format() {
+    let config = ExtractionConfig::default();
+    let json_string = serde_json::to_string(&config).expect("Failed to serialize to string");
+
+    // Validate that output is valid JSON
+    let parsed: serde_json::Value = serde_json::from_str(&json_string).expect("Invalid JSON output");
+    assert!(parsed.is_object(), "JSON should be an object");
+}
+
+#[test]
+fn test_extraction_config_pretty_print() {
+    let config = ExtractionConfig::default();
+    let pretty_json = serde_json::to_string_pretty(&config).expect("Failed to serialize");
+
+    // Validate that pretty-printed JSON is parseable
+    let _parsed: serde_json::Value = serde_json::from_str(&pretty_json).expect("Invalid pretty-printed JSON");
+
+    // Pretty JSON should have newlines
+    assert!(pretty_json.contains('\n'), "Pretty JSON should have newlines");
+}
+
+#[test]
+fn test_extraction_config_field_consistency() {
+    let configs = vec![
+        ExtractionConfig::default(),
+        ExtractionConfig {
+            use_cache: true,
+            ..Default::default()
+        },
+        ExtractionConfig {
+            enable_quality_processing: false,
+            ..Default::default()
+        },
+    ];
+
+    for config in configs {
+        let json = serde_json::to_value(&config).expect("Failed to serialize");
+
+        // All configs should have the same set of top-level fields
+        assert!(json.get("use_cache").is_some());
+        assert!(json.get("enable_quality_processing").is_some());
+        assert!(json.get("force_ocr").is_some());
+    }
+}

data/vendor/kreuzberg/tests/stopwords_integration_test.rs

@@ -67,7 +67,7 @@ fn test_stopwords_removed_during_moderate_token_reduction() {
     };
 
     let input = "The quick brown fox is jumping over the lazy dog and running through the forest";
-    let result = reduce_tokens(input, &config, Some("en")).unwrap();
+    let result = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
 
     assert!(!result.contains(" the "), "Should remove 'the'. Result: {}", result);
     assert!(!result.contains(" is "), "Should remove 'is'. Result: {}", result);
@@ -103,7 +103,7 @@ fn test_stopwords_across_reduction_levels() {
         use_simd: false,
         ..Default::default()
     };
-    let light_result = reduce_tokens(text, &light_config, Some("en")).unwrap();
+    let light_result = reduce_tokens(text, &light_config, Some("en")).expect("Operation failed");
 
     let light_stopwords = count_stopwords(&light_result, "en");
     assert!(light_stopwords > 0, "Light reduction should preserve some stopwords");
@@ -113,7 +113,7 @@ fn test_stopwords_across_reduction_levels() {
         use_simd: false,
         ..Default::default()
     };
-    let moderate_result = reduce_tokens(text, &moderate_config, Some("en")).unwrap();
+    let moderate_result = reduce_tokens(text, &moderate_config, Some("en")).expect("Operation failed");
 
     let moderate_stopwords = count_stopwords(&moderate_result, "en");
     assert!(
@@ -128,7 +128,7 @@ fn test_stopwords_across_reduction_levels() {
         use_simd: false,
         ..Default::default()
     };
-    let aggressive_result = reduce_tokens(text, &aggressive_config, Some("en")).unwrap();
+    let aggressive_result = reduce_tokens(text, &aggressive_config, Some("en")).expect("Operation failed");
 
     assert!(
         aggressive_result.len() <= moderate_result.len(),
@@ -146,7 +146,7 @@ fn test_stopwords_preserve_semantic_meaning() {
 
     let input =
         "The artificial intelligence system is processing the natural language text for extracting meaningful insights";
-    let result = reduce_tokens(input, &config, Some("en")).unwrap();
+    let result = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
 
     let content_words = extract_content_words(&result, "en");
 
@@ -185,7 +185,7 @@ fn test_stopwords_with_multiple_languages() {
         ..Default::default()
     };
     let en_input = "The computer science program is very comprehensive and includes many courses";
-    let en_result = reduce_tokens(en_input, &en_config, Some("en")).unwrap();
+    let en_result = reduce_tokens(en_input, &en_config, Some("en")).expect("Operation failed");
 
     let en_original_stopwords = count_stopwords(en_input, "en");
     let en_result_stopwords = count_stopwords(&en_result, "en");
@@ -200,7 +200,7 @@ fn test_stopwords_with_multiple_languages() {
         ..Default::default()
     };
     let es_input = "El programa de ciencias de la computación es muy completo y tiene muchos cursos";
-    let es_result = reduce_tokens(es_input, &es_config, Some("es")).unwrap();
+    let es_result = reduce_tokens(es_input, &es_config, Some("es")).expect("Operation failed");
 
     let es_original_stopwords = count_stopwords(es_input, "es");
     let es_result_stopwords = count_stopwords(&es_result, "es");
@@ -221,7 +221,7 @@ fn test_stopwords_with_multiple_languages() {
         ..Default::default()
     };
     let de_input = "Die künstliche Intelligenz ist ein wichtiges Forschungsgebiet der Informatik";
-    let de_result = reduce_tokens(de_input, &de_config, Some("de")).unwrap();
+    let de_result = reduce_tokens(de_input, &de_config, Some("de")).expect("Operation failed");
 
     let de_original_stopwords = count_stopwords(de_input, "de");
     let de_result_stopwords = count_stopwords(&de_result, "de");
@@ -240,7 +240,7 @@ fn test_language_fallback_to_english_stopwords() {
     };
 
     let input = "The system is processing the data with the algorithm";
-    let result = reduce_tokens(input, &config, Some("xyz")).unwrap();
+    let result = reduce_tokens(input, &config, Some("xyz")).expect("Operation failed");
 
     let original_stopwords = count_stopwords(input, "en");
     let result_stopwords = count_stopwords(&result, "en");
@@ -267,7 +267,7 @@ fn test_custom_stopwords_integration() {
     };
 
     let input = "The algorithm processes the data in the system efficiently";
-    let result = reduce_tokens(input, &config, Some("en")).unwrap();
+    let result = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
 
     assert!(
         !result.contains("algorithm"),
@@ -301,7 +301,7 @@ fn test_stopwords_with_chinese_text() {
     };
 
     let input = "这个人工智能系统可以处理自然语言";
-    let result = reduce_tokens(input, &config, Some("zh")).unwrap();
+    let result = reduce_tokens(input, &config, Some("zh")).expect("Operation failed");
 
     assert!(
         !result.is_empty(),
@@ -325,7 +325,7 @@ fn test_stopwords_with_mixed_cjk_english() {
     };
 
     let input = "The machine learning model 机器学习模型 is processing data efficiently";
-    let result = reduce_tokens(input, &config, Some("en")).unwrap();
+    let result = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
 
     assert!(
         !result.contains(" the ") && !result.contains("The "),
@@ -355,7 +355,7 @@ fn test_stopwords_with_japanese_text() {
     };
 
     let input = "人工知能技術の研究開発";
-    let result = reduce_tokens(input, &config, Some("ja")).unwrap();
+    let result = reduce_tokens(input, &config, Some("ja")).expect("Operation failed");
 
     assert!(
         !result.is_empty(),
@@ -373,7 +373,7 @@ fn test_stopwords_with_korean_text() {
     };
 
     let input = "인공 지능 기술 개발";
-    let result = reduce_tokens(input, &config, Some("ko")).unwrap();
+    let result = reduce_tokens(input, &config, Some("ko")).expect("Operation failed");
 
     assert!(
         !result.is_empty(),
@@ -391,7 +391,7 @@ fn test_stopwords_excluded_from_rake_keywords() {
 
     let config = KeywordConfig::rake().with_language("en").with_max_keywords(10);
 
-    let keywords = extract_keywords(text, &config).unwrap();
+    let keywords = extract_keywords(text, &config).expect("Operation failed");
 
     assert!(!keywords.is_empty(), "Should extract keywords");
 
@@ -439,7 +439,7 @@ fn test_stopwords_excluded_from_yake_keywords() {
 
     let config = KeywordConfig::yake().with_language("en").with_max_keywords(10);
 
-    let keywords = extract_keywords(text, &config).unwrap();
+    let keywords = extract_keywords(text, &config).expect("Operation failed");
 
     assert!(!keywords.is_empty(), "Should extract keywords");
 
@@ -472,7 +472,7 @@ fn test_keywords_respect_language_specific_stopwords() {
 
     let config = KeywordConfig::rake().with_language("es").with_max_keywords(8);
 
-    let keywords = extract_keywords(spanish_text, &config).unwrap();
+    let keywords = extract_keywords(spanish_text, &config).expect("Operation failed");
 
     assert!(!keywords.is_empty(), "Should extract Spanish keywords");
 
@@ -516,7 +516,7 @@ fn test_all_stopwords_text_reduction() {
     };
 
     let input = "the is a an and or but of to in for on at by";
-    let result = reduce_tokens(input, &config, Some("en")).unwrap();
+    let result = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
 
     assert!(
         result.len() < input.len(),
@@ -533,7 +533,7 @@ fn test_no_stopwords_text_reduction() {
     };
 
     let input = "PyTorch TensorFlow CUDA GPU optimization benchmark performance metrics";
-    let result = reduce_tokens(input, &config, Some("en")).unwrap();
+    let result = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
 
     let input_words: Vec<&str> = input.split_whitespace().collect();
     let result_lower = result.to_lowercase();
@@ -558,7 +558,7 @@ fn test_mixed_case_stopwords_removal() {
     };
 
     let input = "The SYSTEM Is Processing The DATA With The ALGORITHM";
-    let result = reduce_tokens(input, &config, Some("en")).unwrap();
+    let result = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
 
     let result_words: Vec<&str> = result.split_whitespace().collect();
     assert!(
@@ -594,7 +594,7 @@ fn test_reduce_tokens_function_with_stopwords() {
     };
 
     let text = "The artificial intelligence system processes the natural language efficiently";
-    let result = reduce_tokens(text, &config, Some("en")).unwrap();
+    let result = reduce_tokens(text, &config, Some("en")).expect("Operation failed");
 
     let original_stopwords = count_stopwords(text, "en");
     let result_stopwords = count_stopwords(&result, "en");
@@ -622,7 +622,7 @@ fn test_stopwords_with_punctuation() {
     };
 
     let input = "The system, which is processing the data, uses the algorithm.";
-    let result = reduce_tokens(input, &config, Some("en")).unwrap();
+    let result = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
 
     assert!(
         !result.contains(" the ") || result.split_whitespace().filter(|w| w.contains("the")).count() < 3,
@@ -646,7 +646,7 @@ fn test_stopwords_with_numbers() {
     };
 
     let input = "The model has 100 layers and processes the data in 10 seconds";
-    let result = reduce_tokens(input, &config, Some("en")).unwrap();
+    let result = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
 
     assert!(
         result.contains("100"),
@@ -672,9 +672,9 @@ fn test_stopwords_removal_consistency_across_calls() {
 
     let input = "The machine learning model is trained on the dataset";
 
-    let result1 = reduce_tokens(input, &config, Some("en")).unwrap();
-    let result2 = reduce_tokens(input, &config, Some("en")).unwrap();
-    let result3 = reduce_tokens(input, &config, Some("en")).unwrap();
+    let result1 = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
+    let result2 = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
+    let result3 = reduce_tokens(input, &config, Some("en")).expect("Operation failed");
 
     assert_eq!(result1, result2, "Results should be consistent across calls");
     assert_eq!(result2, result3, "Results should be consistent across calls");
@@ -694,7 +694,7 @@ fn test_stopwords_with_long_text() {
                      The system processes the data efficiently and achieves the best performance. ";
     let input = paragraph.repeat(10);
 
-    let result = reduce_tokens(&input, &config, Some("en")).unwrap();
+    let result = reduce_tokens(&input, &config, Some("en")).expect("Operation failed");
 
     assert!(
         result.len() < input.len(),
@@ -719,9 +719,9 @@ fn test_get_stopwords_with_fallback_in_reduction() {
     let primary_stopwords = get_stopwords_with_fallback("xyz", "en");
     assert!(primary_stopwords.is_some(), "Should fallback to English");
 
-    let en_stopwords = get_stopwords("en").unwrap();
+    let en_stopwords = get_stopwords("en").expect("Operation failed");
     assert_eq!(
-        primary_stopwords.unwrap().len(),
+        primary_stopwords.expect("Operation failed").len(),
         en_stopwords.len(),
         "Fallback should return English stopwords"
     );
@@ -733,7 +733,7 @@ fn test_get_stopwords_with_fallback_in_reduction() {
     };
 
     let input = "The system is processing the data";
-    let result = reduce_tokens(input, &config, Some("xyz")).unwrap();
+    let result = reduce_tokens(input, &config, Some("xyz")).expect("Operation failed");
 
     assert!(
         !result.contains(" the ") && !result.contains(" is "),
@@ -789,7 +789,7 @@ fn test_token_reduction_handles_multibyte_utf8() {
     };
 
     let input = "品質管理は重要です。🚀 高速抽出と漢字処理が求められています。";
-    let result = reduce_tokens(input, &config, Some("ja")).unwrap();
+    let result = reduce_tokens(input, &config, Some("ja")).expect("Operation failed");
 
     assert!(
         result.contains("品質管理") || result.contains("漢字処理"),
@@ -814,7 +814,7 @@ fn test_token_reduction_concurrent_access() {
         for _ in 0..8 {
             let cfg = Arc::clone(&config);
             scope.spawn(move || {
-                let reduced = reduce_tokens(input, &cfg, Some("en")).unwrap();
+                let reduced = reduce_tokens(input, &cfg, Some("en")).expect("Operation failed");
                 assert!(!reduced.is_empty());
             });
         }
@@ -831,7 +831,7 @@ fn demo_stopwords_effectiveness() {
         use_simd: false,
         ..Default::default()
     };
-    let en_result = reduce_tokens(en_text, &en_config, Some("en")).unwrap();
+    let en_result = reduce_tokens(en_text, &en_config, Some("en")).expect("Operation failed");
 
     println!("\n=== English Example ===");
     println!("BEFORE: {} chars", en_text.len());
@@ -849,7 +849,7 @@ fn demo_stopwords_effectiveness() {
         use_simd: false,
         ..Default::default()
     };
-    let zh_result = reduce_tokens(zh_text, &zh_config, Some("zh")).unwrap();
+    let zh_result = reduce_tokens(zh_text, &zh_config, Some("zh")).expect("Operation failed");
 
     println!("\n=== Chinese Example ===");
     println!("BEFORE: {}", zh_text);
@@ -870,7 +870,7 @@ fn demo_stopwords_effectiveness() {
             use_simd: false,
             ..Default::default()
         };
-        let result = reduce_tokens(text, &config, Some("en")).unwrap();
+        let result = reduce_tokens(text, &config, Some("en")).expect("Operation failed");
         println!(
             "{:?}: {} chars -> {} chars ({}% reduction)",
             level,
@@ -881,7 +881,7 @@ fn demo_stopwords_effectiveness() {
         println!("  {}", result);
     }
 
-    let stopwords = get_stopwords("en").unwrap();
+    let stopwords = get_stopwords("en").expect("Operation failed");
     println!("\n=== Stopwords Stats ===");
     println!("English stopwords: {}", stopwords.len());
     println!("Sample stopwords: {:?}", stopwords.iter().take(10).collect::<Vec<_>>());