kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/src/plugins/mod.rs

@@ -47,7 +47,6 @@
 //! #             detected_languages: None,
 //! #             chunks: None,
 //! #             images: None,
-//! #             pages: None,
 //! #         })
 //! #     }
 //! #     async fn extract_file(&self, _: &std::path::Path, _: &str, _: &kreuzberg::ExtractionConfig)
@@ -60,7 +59,6 @@
 //! #             detected_languages: None,
 //! #             chunks: None,
 //! #             images: None,
-//! #             pages: None,
 //! #         })
 //! #     }
 //! #     fn supported_mime_types(&self) -> &[&str] { &[] }
@@ -122,7 +120,6 @@
 //!             detected_languages: None,
 //!             chunks: None,
 //!             images: None,
-//!             pages: None,
 //!         })
 //!     }
 //!

data/vendor/kreuzberg/src/plugins/ocr.rs

@@ -10,9 +10,6 @@ use async_trait::async_trait;
 use std::path::Path;
 use std::sync::Arc;
 
-#[cfg(not(feature = "tokio-runtime"))]
-use crate::KreuzbergError;
-
 /// OCR backend types.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum OcrBackendType {
@@ -67,7 +64,6 @@ pub enum OcrBackendType {
 ///             detected_languages: None,
 ///             chunks: None,
 ///             images: None,
-///             pages: None,
 ///         })
 ///     }
 ///
@@ -85,8 +81,7 @@ pub enum OcrBackendType {
 ///     }
 /// }
 /// ```
-#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
-#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[async_trait]
 pub trait OcrBackend: Plugin {
     /// Process an image and extract text via OCR.
     ///
@@ -146,7 +141,6 @@ pub trait OcrBackend: Plugin {
     ///         detected_languages: None,
     ///         chunks: None,
     ///         images: None,
-    ///         pages: None,
     ///     })
     /// }
     /// # }
@@ -167,19 +161,9 @@ pub trait OcrBackend: Plugin {
     ///
     /// Same as `process_image`, plus file I/O errors.
     async fn process_file(&self, path: &Path, config: &OcrConfig) -> Result<ExtractionResult> {
-        #[cfg(feature = "tokio-runtime")]
-        {
-            use crate::core::io;
-            let bytes = io::read_file_async(path).await?;
-            self.process_image(&bytes, config).await
-        }
-        #[cfg(not(feature = "tokio-runtime"))]
-        {
-            let _ = (path, config);
-            Err(KreuzbergError::Other(
-                "File-based OCR processing requires the tokio-runtime feature".to_string(),
-            ))
-        }
+        use crate::core::io;
+        let bytes = io::read_file_async(path).await?;
+        self.process_image(&bytes, config).await
     }
 
     /// Check if this backend supports a given language code.
@@ -268,6 +252,8 @@ pub trait OcrBackend: Plugin {
     }
 }
 
+// Public registration APIs
+
 /// Register an OCR backend with the global registry.
 ///
 /// The OCR backend will be registered with its name from the `name()` method
@@ -317,7 +303,6 @@ pub trait OcrBackend: Plugin {
 ///             detected_languages: None,
 ///             chunks: None,
 ///             images: None,
-///             pages: None,
 ///         })
 ///     }
 ///     fn supports_language(&self, _: &str) -> bool { true }
@@ -335,6 +320,8 @@ pub fn register_ocr_backend(backend: Arc<dyn OcrBackend>) -> crate::Result<()> {
 
     let registry = get_ocr_backend_registry();
     // ~keep: Lock poisoning indicates a panic in another thread holding the lock.
+    // This is a critical runtime error (similar to OOM) that should bubble up
+    // as it indicates the registry is in an inconsistent state.
     let mut registry = registry
         .write()
         .expect("OCR backend registry lock poisoned - critical runtime error");
@@ -370,6 +357,8 @@ pub fn unregister_ocr_backend(name: &str) -> crate::Result<()> {
 
     let registry = get_ocr_backend_registry();
     // ~keep: Lock poisoning indicates a panic in another thread holding the lock.
+    // This is a critical runtime error (similar to OOM) that should bubble up
+    // as it indicates the registry is in an inconsistent state.
     let mut registry = registry
         .write()
         .expect("OCR backend registry lock poisoned - critical runtime error");
@@ -403,6 +392,8 @@ pub fn list_ocr_backends() -> crate::Result<Vec<String>> {
 
     let registry = get_ocr_backend_registry();
     // ~keep: Lock poisoning indicates a panic in another thread holding the lock.
+    // This is a critical runtime error (similar to OOM) that should bubble up
+    // as it indicates the registry is in an inconsistent state.
     let registry = registry
         .read()
         .expect("OCR backend registry lock poisoned - critical runtime error");
@@ -434,6 +425,8 @@ pub fn clear_ocr_backends() -> crate::Result<()> {
 
     let registry = get_ocr_backend_registry();
     // ~keep: Lock poisoning indicates a panic in another thread holding the lock.
+    // This is a critical runtime error (similar to OOM) that should bubble up
+    // as it indicates the registry is in an inconsistent state.
     let mut registry = registry
         .write()
         .expect("OCR backend registry lock poisoned - critical runtime error");
@@ -478,7 +471,6 @@ mod tests {
                 detected_languages: None,
                 chunks: None,
                 images: None,
-                pages: None,
             })
         }
 

data/vendor/kreuzberg/src/plugins/processor.rs

@@ -105,8 +105,7 @@ pub enum ProcessingStage {
 ///     }
 /// }
 /// ```
-#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
-#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[async_trait]
 pub trait PostProcessor: Plugin {
     /// Process an extraction result.
     ///
@@ -373,7 +372,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -423,7 +421,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -490,7 +487,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -516,7 +512,6 @@ mod tests {
                 additional,
                 ..Default::default()
             },
-            pages: None,
             tables: vec![],
             detected_languages: None,
             chunks: None,
@@ -547,7 +542,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         assert_eq!(processor.estimated_duration_ms(&result), 0);
@@ -598,7 +592,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let txt_result = ExtractionResult {
@@ -609,7 +602,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         assert!(processor.should_process(&pdf_result, &config));
@@ -638,7 +630,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let config = ExtractionConfig::default();

data/vendor/kreuzberg/src/plugins/registry.rs

@@ -264,19 +264,10 @@ impl DocumentExtractorRegistry {
     /// # Returns
     ///
     /// The highest priority extractor, or an error if none found.
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self),
-        fields(
-            registry.mime_type = %mime_type,
-            registry.found = tracing::field::Empty,
-        )
-    ))]
     pub fn get(&self, mime_type: &str) -> Result<Arc<dyn DocumentExtractor>> {
         if let Some(priority_map) = self.extractors.get(mime_type)
             && let Some((_priority, extractor)) = priority_map.iter().next_back()
         {
-            #[cfg(feature = "otel")]
-            tracing::Span::current().record("registry.found", true);
             return Ok(Arc::clone(extractor));
         }
 
@@ -302,13 +293,9 @@ impl DocumentExtractorRegistry {
         }
 
         if let Some((_priority, extractor)) = best_match {
-            #[cfg(feature = "otel")]
-            tracing::Span::current().record("registry.found", true);
             return Ok(extractor);
         }
 
-        #[cfg(feature = "otel")]
-        tracing::Span::current().record("registry.found", false);
         Err(KreuzbergError::UnsupportedFormat(mime_type.to_string()))
     }
 
@@ -661,7 +648,6 @@ mod tests {
                 detected_languages: None,
                 chunks: None,
                 images: None,
-                pages: None,
             })
         }
 
@@ -706,7 +692,6 @@ mod tests {
                 detected_languages: None,
                 chunks: None,
                 images: None,
-                pages: None,
             })
         }
 

data/vendor/kreuzberg/src/plugins/validator.rs

@@ -68,8 +68,7 @@ use std::sync::Arc;
 ///     }
 /// }
 /// ```
-#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
-#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[async_trait]
 pub trait Validator: Plugin {
     /// Validate an extraction result.
     ///
@@ -276,6 +275,8 @@ pub trait Validator: Plugin {
     }
 }
 
+// Public registration APIs
+
 /// Register a validator with the global registry.
 ///
 /// The validator will be registered with its default priority and will be called
@@ -489,7 +490,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -508,7 +508,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -529,7 +528,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -565,7 +563,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -613,7 +610,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let txt_result = ExtractionResult {
@@ -624,7 +620,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         assert!(validator.should_validate(&pdf_result, &config));
@@ -708,7 +703,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -736,7 +730,6 @@ mod tests {
                 additional,
                 ..Default::default()
             },
-            pages: None,
             tables: vec![],
             detected_languages: None,
             chunks: None,
@@ -767,7 +760,6 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -796,7 +788,6 @@ mod tests {
                 detected_languages: None,
                 chunks: None,
                 images: None,
-                pages: None,
             };
 
             assert!(validator.validate(&result, &config).await.is_ok());
@@ -815,15 +806,15 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         };
 
         let config = ExtractionConfig::default();
         assert!(validator.validate(&result, &config).await.is_ok());
     }
 
+    // Tests for public registration APIs
+
     #[test]
-    #[serial_test::serial]
     fn test_register_validator() {
         use std::sync::Arc;
 
@@ -835,7 +826,6 @@ mod tests {
     }
 
     #[test]
-    #[serial_test::serial]
     fn test_unregister_validator() {
         use std::sync::Arc;
 
@@ -847,20 +837,19 @@ mod tests {
     }
 
     #[test]
-    #[serial_test::serial]
     fn test_unregister_nonexistent_validator() {
         let result = super::unregister_validator("nonexistent-validator-xyz");
         assert!(result.is_ok());
     }
 
     #[test]
-    #[serial_test::serial]
     fn test_list_validators() {
         use std::sync::Arc;
 
         super::clear_validators().unwrap();
 
         let validator1 = Arc::new(MockValidator { should_fail: false });
+        // Both validators have the same name, so only one will be registered
         let validator2 = Arc::new(MockValidator { should_fail: false });
 
         let list_before = super::list_validators().unwrap();
@@ -870,6 +859,7 @@ mod tests {
         super::register_validator(validator2).unwrap();
 
         let list = super::list_validators().unwrap();
+        // Only 1 validator registered since they have the same name
         assert_eq!(list.len(), 1);
         assert!(list.contains(&"mock-validator".to_string()));
 
@@ -877,7 +867,6 @@ mod tests {
     }
 
     #[test]
-    #[serial_test::serial]
     fn test_clear_validators() {
         use std::sync::Arc;
 
@@ -889,6 +878,7 @@ mod tests {
         super::register_validator(validator1).unwrap();
         super::register_validator(validator2).unwrap();
 
+        // Verify at least one validator is registered
         let list_before = super::list_validators().unwrap();
         assert!(!list_before.is_empty());
 
@@ -900,7 +890,6 @@ mod tests {
     }
 
     #[test]
-    #[serial_test::serial]
     fn test_register_validator_with_invalid_name() {
         use std::sync::Arc;
 
@@ -933,7 +922,6 @@ mod tests {
     }
 
     #[test]
-    #[serial_test::serial]
     fn test_register_validator_with_empty_name() {
         use std::sync::Arc;
 

data/vendor/kreuzberg/src/stopwords/mod.rs

@@ -100,7 +100,7 @@ macro_rules! embed_stopwords {
                         panic!(
                             "Failed to parse embedded stopwords for language '{}': {}. \
                             This indicates corrupted or malformed JSON in the embedded stopwords data. \
-                            Please report this issue at https://github.com/kreuzberg-dev/kreuzberg/issues",
+                            Please report this issue at https://github.com/Goldziher/kreuzberg/issues",
                             $lang, e
                         );
                     }
@@ -1437,7 +1437,7 @@ mod tests {
         let duration = start.elapsed();
 
         assert!(
-            duration.as_millis() < 500,
+            duration.as_millis() < 100,
             "30,000 lookups took too long: {:?}",
             duration
         );

data/vendor/kreuzberg/src/text/mod.rs

@@ -1,5 +1,3 @@
-pub mod utf8_validation;
-
 #[cfg(feature = "quality")]
 pub mod quality;
 
@@ -9,15 +7,9 @@ pub mod string_utils;
 #[cfg(feature = "quality")]
 pub mod token_reduction;
 
-#[cfg(feature = "quality")]
-pub mod quality_processor;
-
 #[cfg(feature = "quality")]
 pub use quality::{calculate_quality_score, clean_extracted_text, normalize_spaces};
 
-#[cfg(feature = "quality")]
-pub use quality_processor::QualityProcessor;
-
 #[cfg(feature = "quality")]
 pub use string_utils::{calculate_text_confidence, fix_mojibake, get_encoding_cache_key, safe_decode};
 

data/vendor/kreuzberg/src/text/quality.rs

@@ -39,23 +39,6 @@ static MALFORMED_WORDS_PATTERN: Lazy<Regex> = Lazy::new(|| {
 static EXCESSIVE_WHITESPACE_PATTERN: Lazy<Regex> =
     Lazy::new(|| Regex::new(r"\s{3,}").expect("Excessive whitespace regex pattern is valid and should compile"));
 
-/// Combined OCR artifact pattern for single-pass scanning (used in calculate_ocr_penalty).
-/// This pattern combines 5 of the 6 OCR patterns with alternation to reduce regex passes
-/// from 5 separate find_iter calls to 1. The dash pattern is handled separately due to
-/// line-based context checking.
-static COMBINED_OCR_ARTIFACTS_PATTERN: Lazy<Regex> = Lazy::new(|| {
-    Regex::new(
-        r"(?x)
-        \b[a-zA-Z]\s{2,}[a-zA-Z]\s{2,}[a-zA-Z]\b |  # Scattered chars
-        [.]{3,}|[_]{3,} |                              # Repeated punctuation
-        \s[.,;:!?]\s |                                 # Isolated punctuation
-        \b[a-zA-Z]+[0-9]+[a-zA-Z]+[a-zA-Z0-9]*\b |   # Malformed words
-        \s{3,}                                        # Excessive whitespace
-    ",
-    )
-    .expect("Combined OCR artifacts regex pattern is valid and should compile")
-});
-
 static JS_FUNCTION_PATTERN: Lazy<Regex> = Lazy::new(|| {
     Regex::new(r"(?i)function\s+\w+\s*\([^)]*\)\s*\{[^}]*\}")
         .expect("JavaScript function regex pattern is valid and should compile")
@@ -123,7 +106,7 @@ where
     }
 }
 
-pub fn calculate_quality_score(text: &str, metadata: Option<&HashMap<String, serde_json::Value>>) -> f64 {
+pub fn calculate_quality_score(text: &str, metadata: Option<&HashMap<String, String>>) -> f64 {
     if text.is_empty() || text.trim().is_empty() {
         return 0.0;
     }
@@ -168,8 +151,12 @@ fn calculate_ocr_penalty(text: &str, total_chars: f64) -> f64 {
         return 0.0;
     }
 
-    let artifact_chars =
-        sum_match_lengths(text, &COMBINED_OCR_ARTIFACTS_PATTERN) + count_non_table_dash_artifacts(text);
+    let artifact_chars = sum_match_lengths(text, &SCATTERED_CHARS_PATTERN)
+        + sum_match_lengths(text, &REPEATED_PUNCT_PATTERN)
+        + count_non_table_dash_artifacts(text)
+        + sum_match_lengths(text, &ISOLATED_PUNCT_PATTERN)
+        + sum_match_lengths(text, &MALFORMED_WORDS_PATTERN)
+        + sum_match_lengths(text, &EXCESSIVE_WHITESPACE_PATTERN);
 
     (artifact_chars as f64 / total_chars).min(1.0)
 }
@@ -266,7 +253,7 @@ fn calculate_structure_bonus(text: &str) -> f64 {
 }
 
 #[inline]
-fn calculate_metadata_bonus(metadata: &HashMap<String, serde_json::Value>) -> f64 {
+fn calculate_metadata_bonus(metadata: &HashMap<String, String>) -> f64 {
     const IMPORTANT_FIELDS: &[&str] = &["title", "author", "subject", "description", "keywords"];
 
     let present_fields = IMPORTANT_FIELDS
@@ -492,8 +479,8 @@ mod tests {
     fn test_calculate_quality_score_with_metadata() {
         let text = "This is a normal text with proper structure.";
         let mut metadata = HashMap::new();
-        metadata.insert("title".to_string(), serde_json::json!("Test Title"));
-        metadata.insert("author".to_string(), serde_json::json!("Test Author"));
+        metadata.insert("title".to_string(), "Test Title".to_string());
+        metadata.insert("author".to_string(), "Test Author".to_string());
 
         let score = calculate_quality_score(text, Some(&metadata));
         assert!(score > 0.0);
@@ -566,11 +553,11 @@ mod tests {
     #[test]
     fn test_calculate_metadata_bonus_full() {
         let mut metadata = HashMap::new();
-        metadata.insert("title".to_string(), serde_json::json!("Title"));
-        metadata.insert("author".to_string(), serde_json::json!("Author"));
-        metadata.insert("subject".to_string(), serde_json::json!("Subject"));
-        metadata.insert("description".to_string(), serde_json::json!("Description"));
-        metadata.insert("keywords".to_string(), serde_json::json!("Keywords"));
+        metadata.insert("title".to_string(), "Title".to_string());
+        metadata.insert("author".to_string(), "Author".to_string());
+        metadata.insert("subject".to_string(), "Subject".to_string());
+        metadata.insert("description".to_string(), "Description".to_string());
+        metadata.insert("keywords".to_string(), "Keywords".to_string());
 
         let bonus = calculate_metadata_bonus(&metadata);
         assert_eq!(bonus, 1.0);

data/vendor/kreuzberg/src/text/string_utils.rs

@@ -45,9 +45,7 @@ fn calculate_cache_key(data: &[u8]) -> String {
     let sample = if data.len() > 1024 { &data[..1024] } else { data };
     sample.hash(&mut hasher);
     data.len().hash(&mut hasher);
-    let mut result = String::with_capacity(16);
-    result.push_str(&format!("{:x}", hasher.finish()));
-    result
+    format!("{:x}", hasher.finish())
 }
 
 pub fn safe_decode(byte_data: &[u8], encoding: Option<&str>) -> String {
@@ -59,7 +57,7 @@ pub fn safe_decode(byte_data: &[u8], encoding: Option<&str>) -> String {
         && let Some(enc) = Encoding::for_label(enc_name.as_bytes())
     {
         let (decoded, _, _) = enc.decode(byte_data);
-        return fix_mojibake_internal(&decoded).into_owned();
+        return fix_mojibake_internal(&decoded);
     }
 
     let cache_key = calculate_cache_key(byte_data);
@@ -68,7 +66,7 @@ pub fn safe_decode(byte_data: &[u8], encoding: Option<&str>) -> String {
         && let Some(&cached_encoding) = cache.get(&cache_key)
     {
         let (decoded, _, _) = cached_encoding.decode(byte_data);
-        return fix_mojibake_internal(&decoded).into_owned();
+        return fix_mojibake_internal(&decoded);
     }
 
     let mut detector = EncodingDetector::new();
@@ -95,23 +93,17 @@ pub fn safe_decode(byte_data: &[u8], encoding: Option<&str>) -> String {
             if let Some(enc) = Encoding::for_label(enc_name.as_bytes()) {
                 let (test_decoded, _, test_errors) = enc.decode(byte_data);
                 if !test_errors && calculate_text_confidence_internal(&test_decoded) > 0.5 {
-                    return fix_mojibake_internal(&test_decoded).into_owned();
+                    return fix_mojibake_internal(&test_decoded);
                 }
             }
         }
     }
 
-    fix_mojibake_internal(&decoded).into_owned()
+    fix_mojibake_internal(&decoded)
 }
 
 pub fn get_encoding_cache_key(data_hash: &str, size: usize) -> String {
-    let estimated_capacity = 16 + 1 + 20;
-    let mut result = String::with_capacity(estimated_capacity);
-    result.push_str(data_hash);
-    result.push(':');
-    use std::fmt::Write;
-    let _ = write!(result, "{}", size);
-    result
+    format!("{}:{}", data_hash, size)
 }
 
 pub fn calculate_text_confidence(text: &str) -> f64 {
@@ -149,16 +141,12 @@ fn calculate_text_confidence_internal(text: &str) -> f64 {
 }
 
 pub fn fix_mojibake(text: &str) -> String {
-    fix_mojibake_internal(text).into_owned()
+    fix_mojibake_internal(text)
 }
 
-fn fix_mojibake_internal(text: &str) -> Cow<'_, str> {
+fn fix_mojibake_internal(text: &str) -> String {
     if text.is_empty() {
-        return Cow::Borrowed("");
-    }
-
-    if !CONTROL_CHARS.is_match(text) && !REPLACEMENT_CHARS.is_match(text) && !ISOLATED_COMBINING.is_match(text) {
-        return Cow::Borrowed(text);
+        return text.to_string();
     }
 
     let replacements = [
@@ -167,7 +155,7 @@ fn fix_mojibake_internal(text: &str) -> Cow<'_, str> {
         (&*ISOLATED_COMBINING, ""),
     ];
 
-    chain_replacements(Cow::Borrowed(text), &replacements)
+    chain_replacements(Cow::Borrowed(text), &replacements).into_owned()
 }
 
 #[cfg(test)]