kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/src/embeddings.rs

@@ -11,30 +11,6 @@
 //! - Batch processing for efficient embedding generation
 //! - Optional GPU acceleration via ONNX Runtime execution providers
 //!
-//! # ONNX Runtime Requirement
-//!
-//! **CRITICAL**: This module requires ONNX Runtime to be installed on the system.
-//! The `embeddings` feature uses dynamic loading (`ort-load-dynamic`), which detects
-//! the ONNX Runtime library at runtime.
-//!
-//! ## Installation Instructions
-//!
-//! - **macOS**: `brew install onnxruntime`
-//! - **Linux (Ubuntu/Debian)**: `apt install libonnxruntime libonnxruntime-dev`
-//! - **Linux (Fedora)**: `dnf install onnxruntime onnxruntime-devel`
-//! - **Linux (Arch)**: `pacman -S onnxruntime`
-//! - **Windows (MSVC)**: Download from https://github.com/microsoft/onnxruntime/releases and add to PATH
-//!
-//! Alternatively, set the `ORT_DYLIB_PATH` environment variable to the ONNX Runtime library path.
-//!
-//! For Docker/containers, install via package manager in your base image.
-//! Verified packages: Ubuntu 22.04+, Fedora 38+, Arch Linux.
-//!
-//! ## Platform Limitations
-//!
-//! **Windows MinGW builds are not supported**. ONNX Runtime requires the MSVC toolchain on Windows.
-//! Please use Windows MSVC builds or disable the embeddings feature.
-//!
 //! # Example
 //!
 //! ```rust,ignore
@@ -67,107 +43,11 @@ use std::sync::{Arc, Mutex, RwLock};
 use std::collections::HashMap;
 
 #[cfg(feature = "embeddings")]
-use std::mem::ManuallyDrop;
+use lazy_static::lazy_static;
 
 #[cfg(feature = "embeddings")]
-use once_cell::sync::Lazy;
-
-/// Wrapper for TextEmbedding that prevents cleanup during process shutdown.
-///
-/// # Problem
-///
-/// ONNX Runtime's C++ destructors fail during process shutdown when trying to
-/// acquire mutexes that have already been torn down by the C++ runtime. This
-/// causes crashes with "mutex lock failed: Invalid argument" errors.
-///
-/// This is a known issue in `ort` v2.0.0-rc.10 (pykeio/ort#441) that was fixed
-/// in later versions, but we're constrained by fastembed's dependency tree.
-///
-/// # Solution
-///
-/// We prevent all cleanup of ONNX Runtime resources:
-/// 1. Individual TextEmbedding objects are leaked via Box::leak
-/// 2. The entire MODEL_CACHE is wrapped in ManuallyDrop
-///
-/// This prevents Drop implementations from running during shutdown, completely
-/// avoiding the mutex errors. The OS reclaims all memory on process exit anyway.
-///
-/// Thread-safe wrapper for leaked TextEmbedding that allows interior mutability.
-///
-/// This wrapper holds a raw pointer to a leaked `TextEmbedding` and provides
-/// safe access through the Mutex lock in MODEL_CACHE.
-#[cfg(feature = "embeddings")]
-pub(crate) struct LeakedModel {
-    ptr: *mut TextEmbedding,
-}
-
-#[cfg(feature = "embeddings")]
-impl LeakedModel {
-    fn new(model: TextEmbedding) -> Self {
-        Self {
-            ptr: Box::into_raw(Box::new(model)),
-        }
-    }
-
-    /// Get a mutable reference to the model.
-    ///
-    /// # Safety
-    ///
-    /// This is safe to call only when:
-    /// 1. The caller has exclusive access (guaranteed by Mutex in MODEL_CACHE)
-    /// 2. The pointer is valid (guaranteed by Box::into_raw and never deallocating)
-    #[allow(unsafe_code, clippy::mut_from_ref)]
-    unsafe fn get_mut(&self) -> &mut TextEmbedding {
-        unsafe { &mut *self.ptr }
-    }
-}
-
-#[cfg(feature = "embeddings")]
-#[allow(unsafe_code)]
-unsafe impl Send for LeakedModel {}
-#[cfg(feature = "embeddings")]
-#[allow(unsafe_code)]
-unsafe impl Sync for LeakedModel {}
-
-#[cfg(feature = "embeddings")]
-type CachedEmbedding = Arc<Mutex<LeakedModel>>;
-
-/// Global model cache wrapped in ManuallyDrop to prevent cleanup during process exit.
-///
-/// We use Lazy + ManuallyDrop because ONNX Runtime's C++ destructors fail during static
-/// destruction when mutexes are already torn down. By never dropping this cache,
-/// we avoid the mutex errors at shutdown. The OS reclaims memory on process exit anyway.
-#[cfg(feature = "embeddings")]
-static MODEL_CACHE: Lazy<ManuallyDrop<RwLock<HashMap<String, CachedEmbedding>>>> =
-    Lazy::new(|| ManuallyDrop::new(RwLock::new(HashMap::new())));
-
-/// Returns installation instructions for ONNX Runtime.
-#[cfg(feature = "embeddings")]
-fn onnx_runtime_install_message() -> String {
-    #[cfg(all(windows, target_env = "gnu"))]
-    {
-        return "ONNX Runtime embeddings are not supported on Windows MinGW builds. \
-        ONNX Runtime requires MSVC toolchain. \
-        Please use Windows MSVC builds or disable embeddings feature."
-            .to_string();
-    }
-
-    #[cfg(not(all(windows, target_env = "gnu")))]
-    {
-        "ONNX Runtime is required for embeddings functionality. \
-        Install: \
-        macOS: 'brew install onnxruntime', \
-        Linux (Ubuntu/Debian): 'apt install libonnxruntime libonnxruntime-dev', \
-        Linux (Fedora): 'dnf install onnxruntime onnxruntime-devel', \
-        Linux (Arch): 'pacman -S onnxruntime', \
-        Windows (MSVC): Download from https://github.com/microsoft/onnxruntime/releases and add to PATH. \
-        \
-        Alternatively, set ORT_DYLIB_PATH environment variable to the ONNX Runtime library path. \
-        \
-        For Docker/containers: Install via package manager in your base image. \
-        Verified packages: Ubuntu 22.04+, Fedora 38+, Arch Linux."
-            .to_string()
-    }
+lazy_static! {
+    static ref MODEL_CACHE: RwLock<HashMap<String, Arc<Mutex<TextEmbedding>>>> = RwLock::new(HashMap::new());
 }
 
 /// Get or initialize a text embedding model from cache.
@@ -175,11 +55,10 @@ fn onnx_runtime_install_message() -> String {
 /// This function ensures models are initialized only once and reused across
 /// the application, avoiding redundant downloads and initialization overhead.
 #[cfg(feature = "embeddings")]
-#[allow(private_interfaces)]
 pub fn get_or_init_model(
     model: EmbeddingModel,
     cache_dir: Option<std::path::PathBuf>,
-) -> crate::Result<CachedEmbedding> {
+) -> crate::Result<Arc<Mutex<TextEmbedding>>> {
     let cache_directory = cache_dir.unwrap_or_else(|| {
         let mut path = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from("."));
         path.push(".kreuzberg");
@@ -190,26 +69,21 @@ pub fn get_or_init_model(
     let model_key = format!("{:?}_{}", model, cache_directory.display());
 
     {
-        match MODEL_CACHE.read() {
-            Ok(cache) => {
-                if let Some(cached_model) = cache.get(&model_key) {
-                    return Ok(Arc::clone(cached_model));
-                }
-            }
-            Err(poison_error) => {
-                let cache = poison_error.get_ref();
-                if let Some(cached_model) = cache.get(&model_key) {
-                    return Ok(Arc::clone(cached_model));
-                }
-            }
+        let cache = MODEL_CACHE.read().map_err(|e| crate::KreuzbergError::Plugin {
+            message: format!("Failed to acquire model cache read lock: {}", e),
+            plugin_name: "embeddings".to_string(),
+        })?;
+
+        if let Some(cached_model) = cache.get(&model_key) {
+            return Ok(Arc::clone(cached_model));
         }
     }
 
     {
-        let mut cache = match MODEL_CACHE.write() {
-            Ok(guard) => guard,
-            Err(poison_error) => poison_error.into_inner(),
-        };
+        let mut cache = MODEL_CACHE.write().map_err(|e| crate::KreuzbergError::Plugin {
+            message: format!("Failed to acquire model cache write lock: {}", e),
+            plugin_name: "embeddings".to_string(),
+        })?;
 
         if let Some(cached_model) = cache.get(&model_key) {
             return Ok(Arc::clone(cached_model));
@@ -218,27 +92,12 @@ pub fn get_or_init_model(
         let mut init_options = InitOptions::new(model);
         init_options = init_options.with_cache_dir(cache_directory);
 
-        let embedding_model = TextEmbedding::try_new(init_options).map_err(|e| {
-            let error_msg = e.to_string();
-
-            if error_msg.contains("onnxruntime")
-                || error_msg.contains("ORT")
-                || error_msg.contains("libonnxruntime")
-                || error_msg.contains("onnxruntime.dll")
-                || error_msg.contains("Unable to load")
-                || error_msg.contains("library load failed")
-            {
-                crate::KreuzbergError::MissingDependency(format!("ONNX Runtime - {}", onnx_runtime_install_message()))
-            } else {
-                crate::KreuzbergError::Plugin {
-                    message: format!("Failed to initialize embedding model: {}", e),
-                    plugin_name: "embeddings".to_string(),
-                }
-            }
+        let embedding_model = TextEmbedding::try_new(init_options).map_err(|e| crate::KreuzbergError::Plugin {
+            message: format!("Failed to initialize embedding model: {}", e),
+            plugin_name: "embeddings".to_string(),
         })?;
 
-        let leaked_model = LeakedModel::new(embedding_model);
-        let arc_model = Arc::new(Mutex::new(leaked_model));
+        let arc_model = Arc::new(Mutex::new(embedding_model));
         cache.insert(model_key, Arc::clone(&arc_model));
 
         Ok(arc_model)
@@ -389,15 +248,12 @@ pub fn generate_embeddings_for_chunks(
     let texts: Vec<String> = chunks.iter().map(|chunk| chunk.content.clone()).collect();
 
     let embeddings_result = {
-        let locked_model = model.lock().map_err(|e| crate::KreuzbergError::Plugin {
+        let mut locked_model = model.lock().map_err(|e| crate::KreuzbergError::Plugin {
             message: format!("Failed to acquire model lock: {}", e),
             plugin_name: "embeddings".to_string(),
         })?;
 
-        #[allow(unsafe_code)]
-        let model_mut = unsafe { locked_model.get_mut() };
-
-        model_mut
+        locked_model
             .embed(texts, Some(config.batch_size))
             .map_err(|e| crate::KreuzbergError::Plugin {
                 message: format!("Failed to generate embeddings: {}", e),
@@ -464,8 +320,4 @@ mod tests {
         assert_eq!(quality.chunk_size, 2000);
         assert_eq!(quality.overlap, 200);
     }
-
-    #[cfg(feature = "embeddings")]
-    #[test]
-    fn test_lock_poisoning_recovery_semantics() {}
 }

data/vendor/kreuzberg/src/error.rs

@@ -60,7 +60,7 @@ pub type Result<T> = std::result::Result<T, KreuzbergError>;
 /// - `Cache` - Cache operation errors (non-fatal, can be ignored)
 /// - `ImageProcessing` - Image manipulation errors
 /// - `Serialization` - JSON/MessagePack serialization errors
-/// - `MissingDependency` - Missing optional dependencies (tesseract, etc.)
+/// - `MissingDependency` - Missing optional dependencies (tesseract, pandoc, etc.)
 /// - `Plugin` - Plugin-specific errors
 /// - `LockPoisoned` - Mutex/RwLock poisoning (should not happen in normal operation)
 /// - `UnsupportedFormat` - Unsupported MIME type or file format
@@ -177,7 +177,7 @@ impl From<crate::pdf::error::PdfError> for KreuzbergError {
 
 macro_rules! error_constructor {
     ($name:ident, $variant:ident) => {
-        pastey::paste! {
+        paste::paste! {
             #[doc = "Create a " $variant " error"]
             pub fn $name<S: Into<String>>(message: S) -> Self {
                 Self::$variant {

data/vendor/kreuzberg/src/extraction/archive.rs

@@ -3,7 +3,7 @@
 //! This module provides functions for extracting file lists and contents from archives.
 
 use crate::error::{KreuzbergError, Result};
-use sevenz_rust2::{ArchiveReader, Password};
+use sevenz_rust::SevenZReader;
 use std::collections::HashMap;
 use std::io::{Cursor, Read};
 use tar::Archive as TarArchive;
@@ -39,7 +39,7 @@ pub fn extract_zip_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
     let mut archive =
         ZipArchive::new(cursor).map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP archive: {}", e)))?;
 
-    let mut file_list = Vec::with_capacity(archive.len());
+    let mut file_list = Vec::new();
     let mut total_size = 0u64;
 
     for i in 0..archive.len() {
@@ -71,8 +71,7 @@ pub fn extract_tar_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
     let cursor = Cursor::new(bytes);
     let mut archive = TarArchive::new(cursor);
 
-    let estimated_entries = bytes.len().saturating_div(512).max(16);
-    let mut file_list = Vec::with_capacity(estimated_entries);
+    let mut file_list = Vec::new();
     let mut total_size = 0u64;
     let mut file_count = 0;
 
@@ -116,8 +115,7 @@ pub fn extract_zip_text_content(bytes: &[u8]) -> Result<HashMap<String, String>>
     let mut archive =
         ZipArchive::new(cursor).map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP archive: {}", e)))?;
 
-    let estimated_text_files = archive.len().saturating_mul(3).saturating_div(10).max(2);
-    let mut contents = HashMap::with_capacity(estimated_text_files);
+    let mut contents = HashMap::new();
     let text_extensions = [
         ".txt", ".md", ".json", ".xml", ".html", ".csv", ".log", ".yaml", ".toml",
     ];
@@ -130,8 +128,7 @@ pub fn extract_zip_text_content(bytes: &[u8]) -> Result<HashMap<String, String>>
         let path = file.name().to_string();
 
         if !file.is_dir() && text_extensions.iter().any(|ext| path.to_lowercase().ends_with(ext)) {
-            let estimated_size = (file.size() as usize).min(10 * 1024 * 1024);
-            let mut content = String::with_capacity(estimated_size);
+            let mut content = String::new();
             if file.read_to_string(&mut content).is_ok() {
                 contents.insert(path, content);
             }
@@ -148,8 +145,7 @@ pub fn extract_tar_text_content(bytes: &[u8]) -> Result<HashMap<String, String>>
     let cursor = Cursor::new(bytes);
     let mut archive = TarArchive::new(cursor);
 
-    let estimated_text_files = bytes.len().saturating_div(1024 * 10).min(100);
-    let mut contents = HashMap::with_capacity(estimated_text_files.max(2));
+    let mut contents = HashMap::new();
     let text_extensions = [
         ".txt", ".md", ".json", ".xml", ".html", ".csv", ".log", ".yaml", ".toml",
     ];
@@ -170,8 +166,7 @@ pub fn extract_tar_text_content(bytes: &[u8]) -> Result<HashMap<String, String>>
 
         if !entry.header().entry_type().is_dir() && text_extensions.iter().any(|ext| path.to_lowercase().ends_with(ext))
         {
-            let estimated_size = (entry.size().min(10 * 1024 * 1024)) as usize;
-            let mut content = String::with_capacity(estimated_size);
+            let mut content = String::new();
             if entry.read_to_string(&mut content).is_ok() {
                 contents.insert(path, content);
             }
@@ -184,7 +179,7 @@ pub fn extract_tar_text_content(bytes: &[u8]) -> Result<HashMap<String, String>>
 /// Extract metadata from a 7z archive.
 pub fn extract_7z_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
     let cursor = Cursor::new(bytes);
-    let archive = ArchiveReader::new(cursor, Password::empty())
+    let archive = SevenZReader::new(cursor, bytes.len() as u64, "".into())
         .map_err(|e| KreuzbergError::parsing(format!("Failed to read 7z archive: {}", e)))?;
 
     let mut file_list = Vec::new();
@@ -217,7 +212,7 @@ pub fn extract_7z_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
 /// Only extracts files with common text extensions: .txt, .md, .json, .xml, .html, .csv, .log
 pub fn extract_7z_text_content(bytes: &[u8]) -> Result<HashMap<String, String>> {
     let cursor = Cursor::new(bytes);
-    let mut archive = ArchiveReader::new(cursor, Password::empty())
+    let mut archive = SevenZReader::new(cursor, bytes.len() as u64, "".into())
         .map_err(|e| KreuzbergError::parsing(format!("Failed to read 7z archive: {}", e)))?;
 
     let mut contents = HashMap::new();
@@ -464,26 +459,26 @@ mod tests {
 
     #[test]
     fn test_extract_7z_metadata_with_files() {
-        use sevenz_rust2::{ArchiveEntry, ArchiveWriter};
+        use sevenz_rust::SevenZWriter;
 
-        let cursor = {
-            let cursor = Cursor::new(Vec::new());
-            let mut sz = ArchiveWriter::new(cursor).unwrap();
+        let mut cursor = Cursor::new(Vec::new());
+        {
+            let mut sz = SevenZWriter::new(&mut cursor).unwrap();
 
             sz.push_archive_entry(
-                ArchiveEntry::new_file("test.txt"),
+                sevenz_rust::SevenZArchiveEntry::from_path("test.txt", "test.txt".to_string()),
                 Some(Cursor::new(b"Hello 7z!".to_vec())),
             )
             .unwrap();
 
             sz.push_archive_entry(
-                ArchiveEntry::new_file("data.json"),
+                sevenz_rust::SevenZArchiveEntry::from_path("data.json", "data.json".to_string()),
                 Some(Cursor::new(b"{\"key\":\"value\"}".to_vec())),
             )
             .unwrap();
 
-            sz.finish().unwrap()
-        };
+            sz.finish().unwrap();
+        }
 
         let bytes = cursor.into_inner();
         let metadata = extract_7z_metadata(&bytes).unwrap();
@@ -839,26 +834,26 @@ mod tests {
 
     #[test]
     fn test_extract_7z_text_content() {
-        use sevenz_rust2::{ArchiveEntry, ArchiveWriter};
+        use sevenz_rust::SevenZWriter;
 
-        let cursor = {
-            let cursor = Cursor::new(Vec::new());
-            let mut sz = ArchiveWriter::new(cursor).unwrap();
+        let mut cursor = Cursor::new(Vec::new());
+        {
+            let mut sz = SevenZWriter::new(&mut cursor).unwrap();
 
             sz.push_archive_entry(
-                ArchiveEntry::new_file("test.txt"),
+                sevenz_rust::SevenZArchiveEntry::from_path("test.txt", "test.txt".to_string()),
                 Some(Cursor::new(b"Hello 7z text!".to_vec())),
             )
             .unwrap();
 
             sz.push_archive_entry(
-                ArchiveEntry::new_file("readme.md"),
+                sevenz_rust::SevenZArchiveEntry::from_path("readme.md", "readme.md".to_string()),
                 Some(Cursor::new(b"# 7z README".to_vec())),
             )
             .unwrap();
 
-            sz.finish().unwrap()
-        };
+            sz.finish().unwrap();
+        }
 
         let bytes = cursor.into_inner();
         let contents = extract_7z_text_content(&bytes).unwrap();
@@ -870,13 +865,13 @@ mod tests {
 
     #[test]
     fn test_extract_7z_empty_archive() {
-        use sevenz_rust2::ArchiveWriter;
+        use sevenz_rust::SevenZWriter;
 
-        let cursor = {
-            let cursor = Cursor::new(Vec::new());
-            let sz = ArchiveWriter::new(cursor).unwrap();
-            sz.finish().unwrap()
-        };
+        let mut cursor = Cursor::new(Vec::new());
+        {
+            let sz = SevenZWriter::new(&mut cursor).unwrap();
+            sz.finish().unwrap();
+        }
 
         let bytes = cursor.into_inner();
         let metadata = extract_7z_metadata(&bytes).unwrap();