kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg-ffi/src/string_intern.rs

@@ -1,568 +0,0 @@
-//! String interning FFI module.
-//!
-//! Provides global string interning for frequently-used strings like MIME types,
-//! language codes, and metadata field names. Reduces memory usage by deduplicating
-//! common strings across multiple extraction results.
-//!
-//! # Benefits
-//!
-//! - 5-10% memory savings for typical workloads
-//! - Reduced allocation overhead for repeated strings
-//! - Faster string comparisons (pointer equality for interned strings)
-//! - Thread-safe with lock-free reads
-//!
-//! # Pre-populated Strings
-//!
-//! Common strings are pre-populated for immediate efficiency:
-//! - MIME types: text/plain, application/pdf, image/png, etc.
-//! - Languages: en, es, fr, de, zh, ja, etc.
-//! - Metadata fields: UTF-8, ISO-8859-1, etc.
-//!
-//! # Usage Pattern
-//!
-//! 1. Intern string: `ptr = kreuzberg_intern_string("application/pdf")`
-//! 2. Use interned pointer (lifetime = until all references freed)
-//! 3. Free when done: `kreuzberg_free_interned_string(ptr)`
-//! 4. Check stats: `kreuzberg_string_intern_stats()`
-//!
-//! # Example (C)
-//!
-//! ```c
-//! const char* mime1 = kreuzberg_intern_string("application/pdf");
-//! const char* mime2 = kreuzberg_intern_string("application/pdf");
-//!
-//! // Same pointer for same string (memory shared)
-//! assert(mime1 == mime2);
-//!
-//! kreuzberg_free_interned_string(mime1);
-//! kreuzberg_free_interned_string(mime2);
-//! ```
-
-use crate::{clear_last_error, set_last_error};
-use std::collections::HashMap;
-use std::ffi::{CStr, CString};
-use std::os::raw::c_char;
-use std::ptr;
-use std::sync::Mutex;
-
-/// Statistics for string interning efficiency tracking.
-#[repr(C)]
-pub struct CStringInternStats {
-    /// Number of unique strings currently interned
-    pub unique_count: usize,
-
-    /// Total number of intern requests
-    pub total_requests: usize,
-
-    /// Number of cache hits (string already interned)
-    pub cache_hits: usize,
-
-    /// Number of cache misses (new string added)
-    pub cache_misses: usize,
-
-    /// Estimated memory saved by deduplication (bytes)
-    pub estimated_memory_saved: usize,
-
-    /// Total memory used by interned strings (bytes)
-    pub total_memory_bytes: usize,
-}
-
-/// Interned string entry with reference counting.
-struct InternedString {
-    /// Owned C string
-    c_string: CString,
-
-    /// Reference count (number of times this string is referenced)
-    ref_count: usize,
-
-    /// Original request count (for memory savings calculation)
-    request_count: usize,
-}
-
-/// Global string interning table.
-struct StringInternTable {
-    /// Map from string content to interned entry
-    strings: HashMap<String, InternedString>,
-
-    /// Total number of intern requests
-    total_requests: usize,
-
-    /// Number of cache hits
-    cache_hits: usize,
-}
-
-impl StringInternTable {
-    /// Create new intern table with pre-populated common strings.
-    fn new() -> Self {
-        let mut table = Self {
-            strings: HashMap::new(),
-            total_requests: 0,
-            cache_hits: 0,
-        };
-
-        let common_mimes = [
-            "text/plain",
-            "application/pdf",
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-            "application/msword",
-            "application/vnd.ms-excel",
-            "application/vnd.ms-powerpoint",
-            "image/png",
-            "image/jpeg",
-            "image/gif",
-            "image/tiff",
-            "text/html",
-            "text/xml",
-            "application/json",
-            "application/zip",
-            "message/rfc822",
-        ];
-
-        for mime in &common_mimes {
-            table.intern_string(mime);
-        }
-
-        let common_langs = [
-            "en", "es", "fr", "de", "zh", "ja", "ko", "pt", "ru", "ar", "hi", "it", "nl", "pl", "tr", "vi", "th", "sv",
-            "da", "fi", "no",
-        ];
-
-        for lang in &common_langs {
-            table.intern_string(lang);
-        }
-
-        let common_encodings = ["UTF-8", "ISO-8859-1", "ASCII", "Windows-1252"];
-
-        for encoding in &common_encodings {
-            table.intern_string(encoding);
-        }
-
-        table.total_requests = 0;
-        table.cache_hits = 0;
-
-        table
-    }
-
-    /// Intern a string and return pointer to C string.
-    fn intern_string(&mut self, s: &str) -> *const c_char {
-        self.total_requests += 1;
-
-        if let Some(entry) = self.strings.get_mut(s) {
-            entry.ref_count += 1;
-            entry.request_count += 1;
-            self.cache_hits += 1;
-            entry.c_string.as_ptr()
-        } else {
-            let c_string = match CString::new(s) {
-                Ok(cs) => cs,
-                Err(_) => return ptr::null(),
-            };
-
-            let ptr = c_string.as_ptr();
-
-            self.strings.insert(
-                s.to_string(),
-                InternedString {
-                    c_string,
-                    ref_count: 1,
-                    request_count: 1,
-                },
-            );
-
-            ptr
-        }
-    }
-
-    /// Free an interned string reference.
-    fn free_string(&mut self, ptr: *const c_char) -> bool {
-        let key = self
-            .strings
-            .iter()
-            .find(|(_, entry)| entry.c_string.as_ptr() == ptr)
-            .map(|(k, _)| k.clone());
-
-        if let Some(key) = key {
-            let entry = self.strings.get_mut(&key).unwrap();
-            entry.ref_count -= 1;
-
-            if entry.ref_count == 0 {
-                self.strings.remove(&key);
-            }
-
-            true
-        } else {
-            false
-        }
-    }
-
-    /// Get statistics about interning efficiency.
-    fn stats(&self) -> CStringInternStats {
-        let total_memory_bytes: usize = self.strings.values().map(|e| e.c_string.as_bytes().len() + 1).sum();
-
-        let estimated_memory_saved: usize = self
-            .strings
-            .values()
-            .map(|e| {
-                if e.request_count > 1 {
-                    (e.request_count - 1) * (e.c_string.as_bytes().len() + 1)
-                } else {
-                    0
-                }
-            })
-            .sum();
-
-        CStringInternStats {
-            unique_count: self.strings.len(),
-            total_requests: self.total_requests,
-            cache_hits: self.cache_hits,
-            cache_misses: self.total_requests - self.cache_hits,
-            estimated_memory_saved,
-            total_memory_bytes,
-        }
-    }
-}
-
-/// Global intern table protected by mutex.
-static INTERN_TABLE: Mutex<Option<StringInternTable>> = Mutex::new(None);
-
-/// Initialize global intern table.
-fn ensure_intern_table() -> &'static Mutex<Option<StringInternTable>> {
-    let mut table = INTERN_TABLE.lock().expect("Mutex poisoned");
-    if table.is_none() {
-        *table = Some(StringInternTable::new());
-    }
-    drop(table);
-    &INTERN_TABLE
-}
-
-/// Intern a string and return pointer to shared C string.
-///
-/// If the string has already been interned, returns pointer to existing allocation.
-/// Otherwise, creates new allocation. Pointer remains valid until all references
-/// are freed with `kreuzberg_free_interned_string()`.
-///
-/// # Arguments
-///
-/// * `s` - Null-terminated UTF-8 string to intern
-///
-/// # Returns
-///
-/// Pointer to interned C string, or NULL on error (invalid UTF-8, allocation failure).
-/// Caller must eventually free with `kreuzberg_free_interned_string()`.
-///
-/// # Reference Counting
-///
-/// Multiple calls with the same string return the same pointer but increment
-/// an internal reference count. The string is freed only when all references
-/// are released.
-///
-/// # Thread Safety
-///
-/// Thread-safe. Multiple threads can call concurrently.
-///
-/// # Safety
-///
-/// - `s` must be valid null-terminated UTF-8 string
-/// - `s` cannot be NULL
-/// - Returned pointer must not be modified
-/// - Caller must call `kreuzberg_free_interned_string()` for each `kreuzberg_intern_string()` call
-///
-/// # Example (C)
-///
-/// ```c
-/// const char* mime1 = kreuzberg_intern_string("application/pdf");
-/// const char* mime2 = kreuzberg_intern_string("application/pdf");
-///
-/// // Same string = same pointer (memory shared)
-/// assert(mime1 == mime2);
-///
-/// // Free each reference
-/// kreuzberg_free_interned_string(mime1);
-/// kreuzberg_free_interned_string(mime2);
-/// ```
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn kreuzberg_intern_string(s: *const c_char) -> *const c_char {
-    clear_last_error();
-
-    if s.is_null() {
-        set_last_error("String cannot be NULL".to_string());
-        return ptr::null();
-    }
-
-    let str_ref = match unsafe { CStr::from_ptr(s) }.to_str() {
-        Ok(s) => s,
-        Err(e) => {
-            set_last_error(format!("Invalid UTF-8: {}", e));
-            return ptr::null();
-        }
-    };
-
-    let table_mutex = ensure_intern_table();
-    let mut table = table_mutex.lock().expect("Mutex poisoned");
-
-    if let Some(ref mut t) = *table {
-        t.intern_string(str_ref)
-    } else {
-        ptr::null()
-    }
-}
-
-/// Free an interned string reference.
-///
-/// Decrements reference count for the interned string. If reference count
-/// reaches zero, the string is freed from the intern table.
-///
-/// # Arguments
-///
-/// * `s` - Pointer returned by `kreuzberg_intern_string()`
-///
-/// # Safety
-///
-/// - `s` must be a pointer returned by `kreuzberg_intern_string()`
-/// - `s` can be NULL (no-op)
-/// - Must not be called twice on same pointer (double-free)
-/// - Pointer becomes invalid after last reference is freed
-///
-/// # Example (C)
-///
-/// ```c
-/// const char* mime = kreuzberg_intern_string("application/pdf");
-/// // Use mime...
-/// kreuzberg_free_interned_string(mime);
-/// // Don't use mime after this point
-/// ```
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn kreuzberg_free_interned_string(s: *const c_char) {
-    if s.is_null() {
-        return;
-    }
-
-    clear_last_error();
-
-    let table_mutex = ensure_intern_table();
-    let mut table = table_mutex.lock().expect("Mutex poisoned");
-
-    if let Some(ref mut t) = *table
-        && !t.free_string(s)
-    {
-        set_last_error("String not found in intern table".to_string());
-    }
-}
-
-/// Get statistics about string interning efficiency.
-///
-/// Returns metrics about unique strings, cache hits/misses, and memory savings.
-///
-/// # Returns
-///
-/// Statistics structure with current metrics.
-///
-/// # Example (C)
-///
-/// ```c
-/// CStringInternStats stats = kreuzberg_string_intern_stats();
-/// printf("Interned: %zu unique strings\n", stats.unique_count);
-/// printf("Requests: %zu total (%zu hits, %zu misses)\n",
-///        stats.total_requests, stats.cache_hits, stats.cache_misses);
-/// printf("Memory saved: %zu bytes\n", stats.estimated_memory_saved);
-/// printf("Hit rate: %.1f%%\n",
-///        100.0 * stats.cache_hits / stats.total_requests);
-/// ```
-#[unsafe(no_mangle)]
-pub extern "C" fn kreuzberg_string_intern_stats() -> CStringInternStats {
-    clear_last_error();
-
-    let table_mutex = ensure_intern_table();
-    let table = table_mutex.lock().expect("Mutex poisoned");
-
-    if let Some(ref t) = *table {
-        t.stats()
-    } else {
-        CStringInternStats {
-            unique_count: 0,
-            total_requests: 0,
-            cache_hits: 0,
-            cache_misses: 0,
-            estimated_memory_saved: 0,
-            total_memory_bytes: 0,
-        }
-    }
-}
-
-/// Reset the intern table, freeing all interned strings.
-///
-/// **WARNING**: This invalidates all pointers returned by `kreuzberg_intern_string()`.
-/// Only use during shutdown or testing.
-///
-/// # Safety
-///
-/// - Must not be called while any interned string pointers are in use
-/// - All existing interned pointers become invalid
-/// - Thread-safe but can race with concurrent intern operations
-#[unsafe(no_mangle)]
-pub extern "C" fn kreuzberg_string_intern_reset() {
-    clear_last_error();
-
-    let table_mutex = ensure_intern_table();
-    let mut table = table_mutex.lock().expect("Mutex poisoned");
-    *table = Some(StringInternTable::new());
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::ffi::CString;
-
-    #[test]
-    fn test_intern_same_string() {
-        let s1 = CString::new("test_unique_12345").unwrap();
-        let s2 = CString::new("test_unique_12345").unwrap();
-
-        unsafe {
-            let stats_before = kreuzberg_string_intern_stats();
-
-            let ptr1 = kreuzberg_intern_string(s1.as_ptr());
-            let ptr2 = kreuzberg_intern_string(s2.as_ptr());
-
-            assert_eq!(ptr1, ptr2);
-
-            let stats = kreuzberg_string_intern_stats();
-            assert!(stats.total_requests - stats_before.total_requests >= 2);
-            assert!(stats.cache_hits - stats_before.cache_hits >= 1);
-
-            kreuzberg_free_interned_string(ptr1);
-            kreuzberg_free_interned_string(ptr2);
-        }
-    }
-
-    #[test]
-    fn test_intern_different_strings() {
-        let s1 = CString::new("test_unique_aaa").unwrap();
-        let s2 = CString::new("test_unique_bbb").unwrap();
-
-        unsafe {
-            let stats_before = kreuzberg_string_intern_stats();
-
-            let ptr1 = kreuzberg_intern_string(s1.as_ptr());
-            let ptr2 = kreuzberg_intern_string(s2.as_ptr());
-
-            assert_ne!(ptr1, ptr2);
-
-            let stats = kreuzberg_string_intern_stats();
-            assert!(stats.total_requests - stats_before.total_requests >= 2);
-            assert!(stats.cache_misses - stats_before.cache_misses >= 2);
-
-            kreuzberg_free_interned_string(ptr1);
-            kreuzberg_free_interned_string(ptr2);
-        }
-    }
-
-    #[test]
-    fn test_intern_reference_counting() {
-        let s = CString::new("test_refcount_xyz").unwrap();
-
-        unsafe {
-            let ptr1 = kreuzberg_intern_string(s.as_ptr());
-            let ptr2 = kreuzberg_intern_string(s.as_ptr());
-            let ptr3 = kreuzberg_intern_string(s.as_ptr());
-
-            let stats_before = kreuzberg_string_intern_stats();
-            let unique_before = stats_before.unique_count;
-
-            kreuzberg_free_interned_string(ptr1);
-            kreuzberg_free_interned_string(ptr2);
-
-            let stats_mid = kreuzberg_string_intern_stats();
-            assert_eq!(stats_mid.unique_count, unique_before);
-
-            kreuzberg_free_interned_string(ptr3);
-
-            let stats_after = kreuzberg_string_intern_stats();
-            assert_eq!(stats_after.unique_count, unique_before - 1);
-        }
-    }
-
-    #[test]
-    fn test_intern_pre_populated() {
-        kreuzberg_string_intern_reset();
-
-        let stats_initial = kreuzberg_string_intern_stats();
-        assert!(stats_initial.unique_count > 0);
-
-        let mime = CString::new("application/pdf").unwrap();
-
-        unsafe {
-            let ptr = kreuzberg_intern_string(mime.as_ptr());
-
-            let stats = kreuzberg_string_intern_stats();
-            assert_eq!(stats.unique_count, stats_initial.unique_count);
-            assert_eq!(stats.cache_hits, 1);
-
-            kreuzberg_free_interned_string(ptr);
-        }
-    }
-
-    #[test]
-    fn test_intern_memory_savings() {
-        let test_str = "test_savings_qwerty";
-        let s = CString::new(test_str).unwrap();
-
-        unsafe {
-            let stats_before = kreuzberg_string_intern_stats();
-
-            let ptr1 = kreuzberg_intern_string(s.as_ptr());
-            let ptr2 = kreuzberg_intern_string(s.as_ptr());
-            let ptr3 = kreuzberg_intern_string(s.as_ptr());
-
-            let stats = kreuzberg_string_intern_stats();
-            let savings_delta = stats.estimated_memory_saved - stats_before.estimated_memory_saved;
-            assert!(savings_delta > 0);
-            assert_eq!(savings_delta, 2 * (test_str.len() + 1));
-
-            kreuzberg_free_interned_string(ptr1);
-            kreuzberg_free_interned_string(ptr2);
-            kreuzberg_free_interned_string(ptr3);
-        }
-    }
-
-    #[test]
-    fn test_intern_null_string() {
-        unsafe {
-            let ptr = kreuzberg_intern_string(ptr::null());
-            assert!(ptr.is_null());
-        }
-    }
-
-    #[test]
-    fn test_free_null_string() {
-        unsafe {
-            kreuzberg_free_interned_string(ptr::null());
-        }
-    }
-
-    #[test]
-    fn test_intern_stats_format() {
-        let s1 = CString::new("test_stats_1").unwrap();
-        let s2 = CString::new("test_stats_2").unwrap();
-
-        unsafe {
-            let stats_before = kreuzberg_string_intern_stats();
-
-            let ptr1 = kreuzberg_intern_string(s1.as_ptr());
-            let _ptr2 = kreuzberg_intern_string(s1.as_ptr());
-            let ptr3 = kreuzberg_intern_string(s2.as_ptr());
-
-            let stats = kreuzberg_string_intern_stats();
-            assert!(stats.unique_count > 0);
-            assert!(stats.total_requests - stats_before.total_requests >= 3);
-            assert!(stats.cache_hits - stats_before.cache_hits >= 1);
-            assert!(stats.cache_misses - stats_before.cache_misses >= 2);
-
-            kreuzberg_free_interned_string(ptr1);
-            kreuzberg_free_interned_string(_ptr2);
-            kreuzberg_free_interned_string(ptr3);
-        }
-    }
-}