kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/cache/utilities.rs

@@ -0,0 +1,156 @@
+//! Cache utilities for key generation and disk space management.
+
+use crate::error::Result;
+use ahash::AHasher;
+use std::hash::{Hash, Hasher};
+
+#[cfg(unix)]
+use crate::error::KreuzbergError;
+#[cfg(unix)]
+use std::path::Path;
+
+/// Cache key hash format width (32 hex digits for u64 hash)
+const CACHE_KEY_HASH_WIDTH: usize = 32;
+
+/// Generate a deterministic cache key from configuration parameters.
+///
+/// # Algorithm
+///
+/// Uses ahash (non-cryptographic 64-bit hash) for performance. Cache keys are
+/// generated by:
+/// 1. Sorting key-value pairs by key (for determinism)
+/// 2. Concatenating as "key1=val1&key2=val2&..."
+/// 3. Hashing with ahash and formatting as 32-character hex
+///
+/// # Collision Probability
+///
+/// AHash produces 64-bit hashes, leading to birthday paradox collisions:
+/// - **~0.01%** probability at 1 million cache entries
+/// - **~1%** probability at 100 million entries
+/// - **~50%** probability at 4.3 billion (2^32) entries
+///
+/// For context: P(collision) ≈ n^2 / (2 * 2^64) where n = number of entries.
+///
+/// # Performance vs Security Trade-off
+///
+/// - **ahash**: ~10x faster than SHA256, sufficient for cache keys
+/// - **SHA256**: Collision-resistant but overkill for caching
+/// - **Practical risk**: Low for typical usage (< 1M entries)
+///
+/// # Impact of Collisions
+///
+/// If two different configurations hash to the same key:
+/// - One configuration reads the other's cached data
+/// - Results in incorrect data served from cache
+/// - Detected via metadata validation (size/mtime checks)
+///
+/// # Recommendations
+///
+/// - **< 1M entries**: ahash is safe and fast
+/// - **> 100M entries**: Monitor cache size, consider periodic clearing
+/// - **Critical data**: If collision risk is unacceptable, add SHA256 option
+///
+/// # Example
+///
+/// ```rust
+/// use kreuzberg::cache::generate_cache_key;
+///
+/// let parts = [("format", "pdf"), ("ocr", "true"), ("lang", "en")];
+/// let key = generate_cache_key(&parts);
+/// assert_eq!(key.len(), 32); // 64-bit hash as hex
+/// ```
+pub fn generate_cache_key(parts: &[(&str, &str)]) -> String {
+    if parts.is_empty() {
+        return "empty".to_string();
+    }
+
+    let mut sorted_parts: Vec<_> = parts.to_vec();
+    sorted_parts.sort_by_key(|(k, _)| *k);
+
+    let estimated_size = sorted_parts.iter().map(|(k, v)| k.len() + v.len() + 2).sum::<usize>();
+    let mut cache_str = String::with_capacity(estimated_size);
+
+    for (i, (key, val)) in sorted_parts.iter().enumerate() {
+        if i > 0 {
+            cache_str.push('&');
+        }
+        cache_str.push_str(&format!("{}={}", key, val));
+    }
+
+    let mut hasher = AHasher::default();
+    cache_str.hash(&mut hasher);
+    let hash = hasher.finish();
+
+    format!("{:0width$x}", hash, width = CACHE_KEY_HASH_WIDTH)
+}
+
+#[allow(unsafe_code)]
+pub fn get_available_disk_space(path: &str) -> Result<f64> {
+    #[cfg(unix)]
+    {
+        let path = Path::new(path);
+        let check_path = if path.exists() {
+            path
+        } else if let Some(parent) = path.parent() {
+            parent
+        } else {
+            Path::new("/")
+        };
+
+        use libc::{statvfs, statvfs as statvfs_struct};
+        use std::ffi::CString;
+
+        let path_str = check_path
+            .to_str()
+            .ok_or_else(|| KreuzbergError::validation("Path contains invalid UTF-8".to_string()))?;
+        let c_path = CString::new(path_str).map_err(|e| KreuzbergError::validation(format!("Invalid path: {}", e)))?;
+
+        let mut stat: statvfs_struct = unsafe { std::mem::zeroed() };
+
+        let result = unsafe { statvfs(c_path.as_ptr(), &mut stat) };
+
+        if result == 0 {
+            #[allow(clippy::unnecessary_cast)]
+            let available_bytes = stat.f_bavail as u64 * stat.f_frsize as u64;
+            Ok(available_bytes as f64 / (1024.0 * 1024.0))
+        } else {
+            tracing::debug!("Failed to get disk stats for {}: errno {}", path_str, result);
+            Ok(10000.0)
+        }
+    }
+
+    #[cfg(not(unix))]
+    {
+        let _ = path;
+        Ok(10000.0)
+    }
+}
+
+pub fn fast_hash(data: &[u8]) -> u64 {
+    let mut hasher = AHasher::default();
+    data.hash(&mut hasher);
+    hasher.finish()
+}
+
+pub fn validate_cache_key(key: &str) -> bool {
+    key.len() == 32 && key.chars().all(|c| c.is_ascii_hexdigit())
+}
+
+pub fn filter_old_cache_entries(cache_times: &[f64], current_time: f64, max_age_seconds: f64) -> Vec<usize> {
+    cache_times
+        .iter()
+        .enumerate()
+        .filter_map(|(idx, &time)| {
+            if current_time - time > max_age_seconds {
+                Some(idx)
+            } else {
+                None
+            }
+        })
+        .collect()
+}
+
+pub fn sort_cache_by_access_time(mut entries: Vec<(String, f64)>) -> Vec<String> {
+    entries.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal));
+    entries.into_iter().map(|(key, _)| key).collect()
+}

data/vendor/kreuzberg/src/chunking/boundaries.rs

@@ -0,0 +1,301 @@
+//! Page boundary handling and page range calculation for chunked text.
+//!
+//! This module provides functions to track which pages text chunks span,
+//! enabling accurate page-level metadata for document processing.
+
+use crate::error::{KreuzbergError, Result};
+use crate::types::PageBoundary;
+
+/// Validates the consistency and correctness of page boundaries.
+///
+/// # Validation Rules
+///
+/// 1. Boundaries must be sorted by byte_start (monotonically increasing)
+/// 2. Boundaries must not overlap (byte_end[i] <= byte_start[i+1])
+/// 3. Each boundary must have byte_start < byte_end
+///
+/// # Arguments
+///
+/// * `boundaries` - Page boundary markers to validate
+///
+/// # Returns
+///
+/// Returns `Ok(())` if all boundaries are valid.
+/// Returns `KreuzbergError::Validation` if any boundary is invalid.
+pub fn validate_page_boundaries(boundaries: &[PageBoundary]) -> Result<()> {
+    if boundaries.is_empty() {
+        return Ok(());
+    }
+
+    for (idx, boundary) in boundaries.iter().enumerate() {
+        if boundary.byte_start >= boundary.byte_end {
+            return Err(KreuzbergError::validation(format!(
+                "Invalid boundary range at index {}: byte_start ({}) must be < byte_end ({})",
+                idx, boundary.byte_start, boundary.byte_end
+            )));
+        }
+    }
+
+    for i in 0..boundaries.len() - 1 {
+        let current = &boundaries[i];
+        let next = &boundaries[i + 1];
+
+        if current.byte_start > next.byte_start {
+            return Err(KreuzbergError::validation(format!(
+                "Page boundaries not sorted: boundary at index {} (byte_start={}) comes after boundary at index {} (byte_start={})",
+                i,
+                current.byte_start,
+                i + 1,
+                next.byte_start
+            )));
+        }
+
+        if current.byte_end > next.byte_start {
+            return Err(KreuzbergError::validation(format!(
+                "Overlapping page boundaries: boundary {} ends at {} but boundary {} starts at {}",
+                i,
+                current.byte_end,
+                i + 1,
+                next.byte_start
+            )));
+        }
+    }
+
+    Ok(())
+}
+
+/// Calculate which pages a byte range spans.
+///
+/// # Arguments
+///
+/// * `byte_start` - Starting byte offset of the chunk
+/// * `byte_end` - Ending byte offset of the chunk
+/// * `boundaries` - Page boundary markers from the document
+///
+/// # Returns
+///
+/// A tuple of (first_page, last_page) where page numbers are 1-indexed.
+/// Returns (None, None) if boundaries are empty or chunk doesn't overlap any page.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if boundaries are invalid.
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// use kreuzberg::chunking::boundaries::calculate_page_range;
+/// use kreuzberg::types::PageBoundary;
+///
+/// let boundaries = vec![
+///     PageBoundary { byte_start: 0, byte_end: 100, page_number: 1 },
+///     PageBoundary { byte_start: 100, byte_end: 200, page_number: 2 },
+/// ];
+///
+/// let (first, last) = calculate_page_range(50, 150, &boundaries)?;
+/// assert_eq!(first, Some(1));
+/// assert_eq!(last, Some(2));
+/// # Ok::<(), kreuzberg::Result<()>>(())
+/// ```
+pub fn calculate_page_range(
+    byte_start: usize,
+    byte_end: usize,
+    boundaries: &[PageBoundary],
+) -> Result<(Option<usize>, Option<usize>)> {
+    if boundaries.is_empty() {
+        return Ok((None, None));
+    }
+
+    validate_page_boundaries(boundaries)?;
+
+    let mut first_page = None;
+    let mut last_page = None;
+
+    for boundary in boundaries {
+        if byte_start < boundary.byte_end && byte_end > boundary.byte_start {
+            if first_page.is_none() {
+                first_page = Some(boundary.page_number);
+            }
+            last_page = Some(boundary.page_number);
+        }
+    }
+
+    Ok((first_page, last_page))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_validate_page_boundaries_valid() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 20,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 20,
+                byte_end: 40,
+                page_number: 2,
+            },
+            PageBoundary {
+                byte_start: 40,
+                byte_end: 60,
+                page_number: 3,
+            },
+        ];
+
+        let result = validate_page_boundaries(&boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_validate_page_boundaries_empty() {
+        let boundaries: Vec<PageBoundary> = vec![];
+        let result = validate_page_boundaries(&boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_calculate_page_range_within_page() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 100,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 100,
+                byte_end: 200,
+                page_number: 2,
+            },
+        ];
+
+        let (first, last) = calculate_page_range(10, 50, &boundaries).unwrap();
+        assert_eq!(first, Some(1));
+        assert_eq!(last, Some(1));
+    }
+
+    #[test]
+    fn test_calculate_page_range_spanning_pages() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 100,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 100,
+                byte_end: 200,
+                page_number: 2,
+            },
+        ];
+
+        let (first, last) = calculate_page_range(50, 150, &boundaries).unwrap();
+        assert_eq!(first, Some(1));
+        assert_eq!(last, Some(2));
+    }
+
+    #[test]
+    fn test_calculate_page_range_empty_boundaries() {
+        let boundaries: Vec<PageBoundary> = vec![];
+
+        let (first, last) = calculate_page_range(0, 50, &boundaries).unwrap();
+        assert_eq!(first, None);
+        assert_eq!(last, None);
+    }
+
+    #[test]
+    fn test_calculate_page_range_no_overlap() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 100,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 100,
+                byte_end: 200,
+                page_number: 2,
+            },
+        ];
+
+        let (first, last) = calculate_page_range(200, 250, &boundaries).unwrap();
+        assert_eq!(first, None);
+        assert_eq!(last, None);
+    }
+
+    #[test]
+    fn test_calculate_page_range_three_pages() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 100,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 100,
+                byte_end: 200,
+                page_number: 2,
+            },
+            PageBoundary {
+                byte_start: 200,
+                byte_end: 300,
+                page_number: 3,
+            },
+        ];
+
+        let (first, last) = calculate_page_range(50, 250, &boundaries).unwrap();
+        assert_eq!(first, Some(1));
+        assert_eq!(last, Some(3));
+    }
+
+    #[test]
+    fn test_calculate_page_range_with_invalid_boundaries() {
+        let boundaries = vec![PageBoundary {
+            byte_start: 15,
+            byte_end: 10,
+            page_number: 1,
+        }];
+
+        let result = calculate_page_range(0, 20, &boundaries);
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(err.to_string().contains("Invalid boundary range"));
+    }
+
+    #[test]
+    fn test_page_boundaries_with_gaps() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 10,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 15,
+                byte_end: 25,
+                page_number: 2,
+            },
+        ];
+
+        let result = validate_page_boundaries(&boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_chunk_with_same_start_and_end() {
+        let boundaries = vec![PageBoundary {
+            byte_start: 10,
+            byte_end: 10,
+            page_number: 1,
+        }];
+
+        let result = validate_page_boundaries(&boundaries);
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(err.to_string().contains("Invalid boundary range"));
+    }
+}