kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg-ffi/src/result_pool.rs

@@ -1,639 +0,0 @@
-//! Result pooling FFI module.
-//!
-//! Provides memory pool for ExtractionResult allocations to reduce overhead
-//! from repeated allocations. Pre-allocates memory and reuses it across
-//! multiple extraction operations.
-//!
-//! # Benefits
-//!
-//! - 15-25% allocation overhead reduction for repeated extractions
-//! - Reduced memory fragmentation
-//! - Predictable memory usage patterns
-//! - Thread-safe with lock-free fast path
-//!
-//! # Usage Pattern
-//!
-//! 1. Create pool with estimated capacity: `kreuzberg_result_pool_new(100)`
-//! 2. Extract files into pool: `kreuzberg_extract_file_into_pool(..., pool)`
-//! 3. Process results (they remain in pool until reset/freed)
-//! 4. Reset pool to reuse: `kreuzberg_result_pool_reset(pool)`
-//! 5. Free pool when done: `kreuzberg_result_pool_free(pool)`
-//!
-//! # Example (C)
-//!
-//! ```c
-//! CResultPool* pool = kreuzberg_result_pool_new(100);
-//!
-//! // Process batch 1
-//! for (int i = 0; i < 50; i++) {
-//!     const CExtractionResult* result = kreuzberg_extract_file_into_pool(
-//!         files[i], NULL, pool
-//!     );
-//!     // Process result
-//! }
-//!
-//! // Reset and reuse for batch 2
-//! kreuzberg_result_pool_reset(pool);
-//!
-//! for (int i = 0; i < 50; i++) {
-//!     const CExtractionResult* result = kreuzberg_extract_file_into_pool(
-//!         files[i + 50], NULL, pool
-//!     );
-//!     // Process result
-//! }
-//!
-//! kreuzberg_result_pool_free(pool);
-//! ```
-
-use crate::result_view::{CExtractionResultView, create_result_view};
-use crate::{FfiResult, clear_last_error, parse_extraction_config_from_json, set_last_error};
-use kreuzberg::types::ExtractionResult;
-use std::ffi::CStr;
-use std::os::raw::c_char;
-use std::path::Path;
-use std::ptr;
-use std::sync::Mutex;
-use std::sync::atomic::{AtomicUsize, Ordering};
-
-/// Statistics for result pool allocation tracking.
-///
-/// Provides insight into pool efficiency and memory usage patterns.
-#[repr(C)]
-pub struct CResultPoolStats {
-    /// Current number of results stored in pool
-    pub current_count: usize,
-
-    /// Maximum capacity of pool (before automatic growth)
-    pub capacity: usize,
-
-    /// Total number of allocations (successful extractions)
-    pub total_allocations: usize,
-
-    /// Number of times pool capacity was exceeded (triggered growth)
-    pub growth_events: usize,
-
-    /// Estimated memory used by results in bytes
-    pub estimated_memory_bytes: usize,
-}
-
-/// Memory pool for ExtractionResult objects.
-///
-/// Pre-allocates storage and reuses memory across multiple extractions.
-/// Thread-safe with internal synchronization.
-///
-/// # Memory Model
-///
-/// - Results are owned by the pool until reset or freed
-/// - Pool grows automatically if capacity is exceeded
-/// - Reset clears all results but retains capacity
-/// - Free releases all memory and destroys pool
-///
-/// # Thread Safety
-///
-/// Pool uses internal Mutex for synchronization. Safe for concurrent access
-/// but may serialize extractions. For parallel processing, consider using
-/// separate pools per thread.
-pub struct ResultPool {
-    /// Storage for extraction results
-    results: Mutex<Vec<ExtractionResult>>,
-
-    /// Initial capacity (for growth tracking)
-    _initial_capacity: usize,
-
-    /// Statistics tracking
-    total_allocations: AtomicUsize,
-    growth_events: AtomicUsize,
-}
-
-impl ResultPool {
-    /// Create new result pool with specified capacity.
-    ///
-    /// Pre-allocates Vec capacity but does not pre-allocate result objects.
-    fn new(capacity: usize) -> Self {
-        Self {
-            results: Mutex::new(Vec::with_capacity(capacity)),
-            _initial_capacity: capacity,
-            total_allocations: AtomicUsize::new(0),
-            growth_events: AtomicUsize::new(0),
-        }
-    }
-
-    /// Add result to pool and return borrowed reference.
-    ///
-    /// Reference is valid until pool is reset or freed.
-    fn add_result(&self, result: ExtractionResult) -> *const ExtractionResult {
-        let mut results = self.results.lock().expect("Mutex poisoned");
-
-        if results.len() == results.capacity() && results.capacity() > 0 {
-            self.growth_events.fetch_add(1, Ordering::Relaxed);
-        }
-
-        results.push(result);
-        self.total_allocations.fetch_add(1, Ordering::Relaxed);
-
-        results.last().unwrap() as *const ExtractionResult
-    }
-
-    /// Clear all results but retain capacity.
-    fn reset(&self) {
-        let mut results = self.results.lock().expect("Mutex poisoned");
-        results.clear();
-    }
-
-    /// Get current statistics.
-    fn stats(&self) -> CResultPoolStats {
-        let results = self.results.lock().expect("Mutex poisoned");
-
-        let estimated_memory_bytes = results
-            .iter()
-            .map(|r| {
-                r.content.len()
-                    + r.mime_type.len()
-                    + r.metadata.title.as_ref().map_or(0, |s| s.len())
-                    + r.metadata
-                        .authors
-                        .as_ref()
-                        .map_or(0, |v| v.iter().map(|s| s.len()).sum())
-                    + r.metadata.subject.as_ref().map_or(0, |s| s.len())
-                    + r.metadata.created_at.as_ref().map_or(0, |s| s.len())
-                    + r.metadata.language.as_ref().map_or(0, |s| s.len())
-            })
-            .sum();
-
-        CResultPoolStats {
-            current_count: results.len(),
-            capacity: results.capacity(),
-            total_allocations: self.total_allocations.load(Ordering::Relaxed),
-            growth_events: self.growth_events.load(Ordering::Relaxed),
-            estimated_memory_bytes,
-        }
-    }
-}
-
-/// Create a new result pool with specified initial capacity.
-///
-/// Pre-allocates storage for `capacity` results to reduce allocation overhead.
-/// Pool automatically grows if capacity is exceeded.
-///
-/// # Arguments
-///
-/// * `capacity` - Initial capacity (number of results to pre-allocate storage for)
-///
-/// # Returns
-///
-/// Pointer to allocated pool, or NULL on allocation failure (check `kreuzberg_last_error`).
-///
-/// # Memory Management
-///
-/// Caller must free the returned pool with `kreuzberg_result_pool_free()`.
-///
-/// # Example (C)
-///
-/// ```c
-/// CResultPool* pool = kreuzberg_result_pool_new(100);
-/// if (pool == NULL) {
-///     fprintf(stderr, "Failed to create pool: %s\n", kreuzberg_last_error());
-///     return;
-/// }
-/// // Use pool...
-/// kreuzberg_result_pool_free(pool);
-/// ```
-#[unsafe(no_mangle)]
-pub extern "C" fn kreuzberg_result_pool_new(capacity: usize) -> *mut ResultPool {
-    clear_last_error();
-
-    let pool = Box::new(ResultPool::new(capacity));
-    Box::into_raw(pool)
-}
-
-/// Reset pool by clearing all results.
-///
-/// Removes all results from the pool but retains allocated capacity.
-/// After reset, pool can be reused for new extractions.
-///
-/// # Arguments
-///
-/// * `pool` - Pointer to result pool
-///
-/// # Safety
-///
-/// - `pool` must be a valid pointer returned by `kreuzberg_result_pool_new()`
-/// - `pool` cannot be NULL
-/// - All result pointers obtained from this pool become invalid after reset
-/// - Must not be called concurrently with extractions using same pool
-///
-/// # Example (C)
-///
-/// ```c
-/// CResultPool* pool = kreuzberg_result_pool_new(100);
-///
-/// // Process batch 1
-/// for (int i = 0; i < 50; i++) {
-///     kreuzberg_extract_file_into_pool(files[i], NULL, pool);
-/// }
-///
-/// // Reset and reuse
-/// kreuzberg_result_pool_reset(pool);
-///
-/// // Process batch 2
-/// for (int i = 0; i < 50; i++) {
-///     kreuzberg_extract_file_into_pool(other_files[i], NULL, pool);
-/// }
-///
-/// kreuzberg_result_pool_free(pool);
-/// ```
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn kreuzberg_result_pool_reset(pool: *mut ResultPool) {
-    clear_last_error();
-
-    if pool.is_null() {
-        set_last_error("Pool cannot be NULL".to_string());
-        return;
-    }
-
-    let pool_ref = unsafe { &*pool };
-    pool_ref.reset();
-}
-
-/// Free result pool and all contained results.
-///
-/// Releases all memory associated with the pool. All result pointers
-/// obtained from this pool become invalid.
-///
-/// # Arguments
-///
-/// * `pool` - Pointer to result pool
-///
-/// # Safety
-///
-/// - `pool` must be a valid pointer returned by `kreuzberg_result_pool_new()`
-/// - `pool` can be NULL (no-op)
-/// - All result pointers from this pool become invalid after free
-/// - Must not be called twice on same pool (double-free)
-/// - Must not be called concurrently with other pool operations
-///
-/// # Example (C)
-///
-/// ```c
-/// CResultPool* pool = kreuzberg_result_pool_new(100);
-/// // Use pool...
-/// kreuzberg_result_pool_free(pool);
-/// pool = NULL; // Prevent double-free
-/// ```
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn kreuzberg_result_pool_free(pool: *mut ResultPool) {
-    if pool.is_null() {
-        return;
-    }
-
-    let _ = unsafe { Box::from_raw(pool) };
-}
-
-/// Get statistics about pool usage and efficiency.
-///
-/// Returns metrics about current pool state, allocation counts, and memory usage.
-///
-/// # Arguments
-///
-/// * `pool` - Pointer to result pool
-///
-/// # Returns
-///
-/// Statistics structure with current metrics, or zeroed structure on error.
-///
-/// # Safety
-///
-/// - `pool` must be a valid pointer returned by `kreuzberg_result_pool_new()`
-/// - `pool` cannot be NULL
-///
-/// # Example (C)
-///
-/// ```c
-/// CResultPoolStats stats = kreuzberg_result_pool_stats(pool);
-/// printf("Pool: %zu/%zu results, %zu allocations, %zu bytes\n",
-///        stats.current_count, stats.capacity,
-///        stats.total_allocations, stats.estimated_memory_bytes);
-///
-/// if (stats.growth_events > 0) {
-///     printf("Warning: Pool grew %zu times (consider larger initial capacity)\n",
-///            stats.growth_events);
-/// }
-/// ```
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn kreuzberg_result_pool_stats(pool: *const ResultPool) -> CResultPoolStats {
-    if pool.is_null() {
-        set_last_error("Pool cannot be NULL".to_string());
-        return CResultPoolStats {
-            current_count: 0,
-            capacity: 0,
-            total_allocations: 0,
-            growth_events: 0,
-            estimated_memory_bytes: 0,
-        };
-    }
-
-    clear_last_error();
-
-    let pool_ref = unsafe { &*pool };
-    pool_ref.stats()
-}
-
-/// Extract file and store result in pool.
-///
-/// Extracts document content and adds result to pool. Returns borrowed reference
-/// to result that remains valid until pool is reset or freed.
-///
-/// # Arguments
-///
-/// * `file_path` - Null-terminated UTF-8 file path
-/// * `config_json` - Optional JSON configuration string (NULL for defaults)
-/// * `pool` - Pointer to result pool
-///
-/// # Returns
-///
-/// Borrowed pointer to extraction result view, or NULL on error (check `kreuzberg_last_error`).
-/// Result remains valid until pool is reset or freed.
-///
-/// # Safety
-///
-/// - `file_path` must be valid null-terminated UTF-8 string
-/// - `config_json` must be valid null-terminated UTF-8 if not NULL
-/// - `pool` must be valid pointer returned by `kreuzberg_result_pool_new()`
-/// - None can be NULL (except config_json which is optional)
-/// - Returned pointer is borrowed from pool (do not free separately)
-/// - Returned pointer becomes invalid when pool is reset or freed
-///
-/// # Example (C)
-///
-/// ```c
-/// CResultPool* pool = kreuzberg_result_pool_new(100);
-///
-/// const CExtractionResultView* result = kreuzberg_extract_file_into_pool(
-///     "document.pdf", NULL, pool
-/// );
-///
-/// if (result != NULL) {
-///     // Access result fields
-///     printf("Content length: %zu\n", result->content_len);
-///     printf("MIME type: %.*s\n",
-///            (int)result->mime_type_len,
-///            result->mime_type_ptr);
-/// }
-///
-/// // Result remains valid until pool is reset/freed
-/// kreuzberg_result_pool_free(pool);
-/// ```
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn kreuzberg_extract_file_into_pool(
-    file_path: *const c_char,
-    config_json: *const c_char,
-    pool: *mut ResultPool,
-) -> *const CExtractionResultView {
-    clear_last_error();
-
-    if file_path.is_null() {
-        set_last_error("File path cannot be NULL".to_string());
-        return ptr::null();
-    }
-
-    if pool.is_null() {
-        set_last_error("Pool cannot be NULL".to_string());
-        return ptr::null();
-    }
-
-    let path_str = match unsafe { CStr::from_ptr(file_path) }.to_str() {
-        Ok(s) => s,
-        Err(e) => {
-            set_last_error(format!("Invalid UTF-8 in file path: {}", e));
-            return ptr::null();
-        }
-    };
-
-    let config = if !config_json.is_null() {
-        match unsafe { CStr::from_ptr(config_json) }.to_str() {
-            Ok(config_str) => match parse_extraction_config_from_json(config_str) {
-                Ok(cfg) => cfg,
-                Err(e) => {
-                    set_last_error(format!("Invalid configuration: {}", e));
-                    return ptr::null();
-                }
-            },
-            Err(e) => {
-                set_last_error(format!("Invalid UTF-8 in config: {}", e));
-                return ptr::null();
-            }
-        }
-    } else {
-        Default::default()
-    };
-
-    let result = match extract_file_internal(path_str, &config) {
-        Ok(r) => r,
-        Err(e) => {
-            set_last_error(e);
-            return ptr::null();
-        }
-    };
-
-    let pool_ref = unsafe { &*pool };
-    let result_ptr = pool_ref.add_result(result);
-
-    result_ptr as *const CExtractionResultView
-}
-
-/// Extract file into pool and get zero-copy view.
-///
-/// Convenience function that combines extraction and view creation.
-/// Equivalent to `kreuzberg_extract_file_into_pool()` followed by
-/// `kreuzberg_get_result_view()`.
-///
-/// # Arguments
-///
-/// Same as `kreuzberg_extract_file_into_pool()`
-///
-/// # Returns
-///
-/// Zero-copy view of result, or zeroed view on error.
-///
-/// # Safety
-///
-/// Same requirements as `kreuzberg_extract_file_into_pool()`.
-/// View is valid until pool is reset or freed.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn kreuzberg_extract_file_into_pool_view(
-    file_path: *const c_char,
-    config_json: *const c_char,
-    pool: *mut ResultPool,
-) -> CExtractionResultView {
-    clear_last_error();
-
-    if file_path.is_null() || pool.is_null() {
-        set_last_error("Arguments cannot be NULL".to_string());
-        return unsafe { std::mem::zeroed() };
-    }
-
-    let path_str = match unsafe { CStr::from_ptr(file_path) }.to_str() {
-        Ok(s) => s,
-        Err(e) => {
-            set_last_error(format!("Invalid UTF-8 in file path: {}", e));
-            return unsafe { std::mem::zeroed() };
-        }
-    };
-
-    let config = if !config_json.is_null() {
-        match unsafe { CStr::from_ptr(config_json) }.to_str() {
-            Ok(config_str) => match parse_extraction_config_from_json(config_str) {
-                Ok(cfg) => cfg,
-                Err(e) => {
-                    set_last_error(format!("Invalid configuration: {}", e));
-                    return unsafe { std::mem::zeroed() };
-                }
-            },
-            Err(e) => {
-                set_last_error(format!("Invalid UTF-8 in config: {}", e));
-                return unsafe { std::mem::zeroed() };
-            }
-        }
-    } else {
-        Default::default()
-    };
-
-    let result = match extract_file_internal(path_str, &config) {
-        Ok(r) => r,
-        Err(e) => {
-            set_last_error(e);
-            return unsafe { std::mem::zeroed() };
-        }
-    };
-
-    let pool_ref = unsafe { &*pool };
-    let result_ptr = pool_ref.add_result(result);
-
-    create_result_view(unsafe { &*result_ptr })
-}
-
-/// Internal extraction function.
-fn extract_file_internal(
-    file_path: &str,
-    config: &kreuzberg::core::config::ExtractionConfig,
-) -> FfiResult<ExtractionResult> {
-    let path = Path::new(file_path);
-
-    if !path.exists() {
-        return Err(format!("File not found: {}", file_path));
-    }
-
-    let rt = tokio::runtime::Runtime::new().map_err(|e| format!("Failed to create runtime: {}", e))?;
-
-    rt.block_on(async {
-        kreuzberg::core::extractor::extract_file(path, None, config)
-            .await
-            .map_err(|e| format!("Extraction failed: {}", e))
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::ffi::CString;
-
-    #[test]
-    fn test_pool_creation() {
-        let pool = kreuzberg_result_pool_new(10);
-        assert!(!pool.is_null());
-
-        unsafe {
-            let stats = kreuzberg_result_pool_stats(pool);
-            assert_eq!(stats.current_count, 0);
-            assert_eq!(stats.capacity, 10);
-            assert_eq!(stats.total_allocations, 0);
-
-            kreuzberg_result_pool_free(pool);
-        }
-    }
-
-    #[test]
-    fn test_pool_extraction() {
-        let temp_dir = tempfile::tempdir().unwrap();
-        let file_path = temp_dir.path().join("test.txt");
-        std::fs::write(&file_path, "Test content").unwrap();
-
-        let pool = kreuzberg_result_pool_new(10);
-        assert!(!pool.is_null());
-
-        let path_cstr = CString::new(file_path.to_str().unwrap()).unwrap();
-
-        unsafe {
-            let view = kreuzberg_extract_file_into_pool_view(path_cstr.as_ptr(), ptr::null(), pool);
-
-            assert!(view.content_len > 0);
-            assert!(!view.content_ptr.is_null());
-
-            let stats = kreuzberg_result_pool_stats(pool);
-            assert_eq!(stats.current_count, 1);
-            assert_eq!(stats.total_allocations, 1);
-
-            kreuzberg_result_pool_free(pool);
-        }
-    }
-
-    #[test]
-    fn test_pool_reset() {
-        let temp_dir = tempfile::tempdir().unwrap();
-        let file_path = temp_dir.path().join("test.txt");
-        std::fs::write(&file_path, "Test content").unwrap();
-
-        let pool = kreuzberg_result_pool_new(10);
-        let path_cstr = CString::new(file_path.to_str().unwrap()).unwrap();
-
-        unsafe {
-            kreuzberg_extract_file_into_pool_view(path_cstr.as_ptr(), ptr::null(), pool);
-
-            let stats_before = kreuzberg_result_pool_stats(pool);
-            assert_eq!(stats_before.current_count, 1);
-
-            kreuzberg_result_pool_reset(pool);
-
-            let stats_after = kreuzberg_result_pool_stats(pool);
-            assert_eq!(stats_after.current_count, 0);
-            assert_eq!(stats_after.total_allocations, 1);
-
-            kreuzberg_result_pool_free(pool);
-        }
-    }
-
-    #[test]
-    fn test_pool_growth() {
-        let temp_dir = tempfile::tempdir().unwrap();
-
-        let pool = kreuzberg_result_pool_new(2);
-
-        unsafe {
-            for i in 0..5 {
-                let file_path = temp_dir.path().join(format!("test{}.txt", i));
-                std::fs::write(&file_path, format!("Content {}", i)).unwrap();
-
-                let path_cstr = CString::new(file_path.to_str().unwrap()).unwrap();
-                kreuzberg_extract_file_into_pool_view(path_cstr.as_ptr(), ptr::null(), pool);
-            }
-
-            let stats = kreuzberg_result_pool_stats(pool);
-            assert_eq!(stats.current_count, 5);
-            assert_eq!(stats.total_allocations, 5);
-            assert!(stats.growth_events > 0);
-
-            kreuzberg_result_pool_free(pool);
-        }
-    }
-
-    #[test]
-    fn test_pool_null_arguments() {
-        unsafe {
-            kreuzberg_result_pool_reset(ptr::null_mut());
-            kreuzberg_result_pool_free(ptr::null_mut());
-
-            let stats = kreuzberg_result_pool_stats(ptr::null());
-            assert_eq!(stats.current_count, 0);
-        }
-    }
-}