kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/utils/quality/patterns.rs

@@ -0,0 +1,117 @@
+//! Regex patterns for quality detection
+//!
+//! This module contains all regex patterns used for detecting OCR artifacts,
+//! script content, navigation elements, and text structure.
+
+use once_cell::sync::Lazy;
+use regex::Regex;
+
+// ============================================================================
+// OCR Artifact Patterns
+// ============================================================================
+
+/// Detects scattered characters with excessive spacing (e.g., "a  b  c")
+pub(crate) static SCATTERED_CHARS_PATTERN: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"\b[a-zA-Z]\s{2,}[a-zA-Z]\s{2,}[a-zA-Z]\b")
+        .expect("Scattered chars regex pattern is valid and should compile")
+});
+
+/// Detects repeated punctuation marks (3 or more dots or underscores)
+pub(crate) static REPEATED_PUNCT_PATTERN: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"[.]{3,}|[_]{3,}").expect("Repeated punctuation regex pattern is valid and should compile")
+});
+
+/// Detects repeated dashes (3 or more)
+pub(crate) static DASH_PATTERN: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"[-]{3,}").expect("Dash pattern regex is valid and should compile"));
+
+/// Detects isolated punctuation surrounded by spaces
+pub(crate) static ISOLATED_PUNCT_PATTERN: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"\s[.,;:!?]\s").expect("Isolated punctuation regex pattern is valid and should compile"));
+
+/// Detects malformed words with mixed letters and numbers
+pub(crate) static MALFORMED_WORDS_PATTERN: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"\b[a-zA-Z]+[0-9]+[a-zA-Z]+[a-zA-Z0-9]*\b")
+        .expect("Malformed words regex pattern is valid and should compile")
+});
+
+/// Detects excessive whitespace (3 or more spaces)
+pub(crate) static EXCESSIVE_WHITESPACE_PATTERN: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"\s{3,}").expect("Excessive whitespace regex pattern is valid and should compile"));
+
+// ============================================================================
+// Script and Code Patterns
+// ============================================================================
+
+/// Detects JavaScript function declarations
+pub(crate) static JS_FUNCTION_PATTERN: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"(?i)function\s+\w+\s*\([^)]*\)\s*\{[^}]*\}")
+        .expect("JavaScript function regex pattern is valid and should compile")
+});
+
+/// Detects CSS rules
+pub(crate) static CSS_RULES_PATTERN: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"(?i)\.[a-zA-Z][\w-]*\s*\{[^}]*\}").expect("CSS rules regex pattern is valid and should compile")
+});
+
+/// Detects HTML script tags
+pub(crate) static SCRIPT_TAG_PATTERN: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"(?is)<script[^>]*>.*?</script>").expect("Script tag regex pattern is valid and should compile")
+});
+
+/// Detects HTML style tags
+pub(crate) static STYLE_TAG_PATTERN: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"(?is)<style[^>]*>.*?</style>").expect("Style tag regex pattern is valid and should compile")
+});
+
+// ============================================================================
+// Navigation Element Patterns
+// ============================================================================
+
+/// Detects common navigation words and phrases
+pub(crate) static NAV_WORDS_PATTERN: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"(?i)\b(?:Skip to main content|Back to top|Main navigation|Site navigation)\b")
+        .expect("Navigation words regex pattern is valid and should compile")
+});
+
+/// Detects breadcrumb navigation patterns
+pub(crate) static BREADCRUMB_PATTERN: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"(?:Home\s*[>»]\s*|[>»]\s*){2,}").expect("Breadcrumb regex pattern is valid and should compile")
+});
+
+/// Detects pagination text
+pub(crate) static PAGINATION_PATTERN: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"(?i)\b(?:Page \d+ of \d+|First page|Last page|Previous page|Next page|^\d+ of \d+$)\b")
+        .expect("Pagination regex pattern is valid and should compile")
+});
+
+// ============================================================================
+// Text Structure Patterns
+// ============================================================================
+
+/// Detects sentence boundaries
+pub(crate) static SENTENCE_DETECT: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"[.!?]\s+[A-Z]").expect("Sentence detection regex pattern is valid and should compile"));
+
+/// Detects punctuation marks
+pub(crate) static PUNCTUATION_DETECT: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"[.!?]").expect("Punctuation detection regex pattern is valid and should compile"));
+
+// ============================================================================
+// Whitespace Normalization Patterns
+// ============================================================================
+
+/// Normalizes various types of whitespace characters
+pub(crate) static WHITESPACE_NORMALIZE: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"[ \t\f\v\r\xa0\u{2000}-\u{200b}\u{2028}\u{2029}\u{3000}]+")
+        .expect("Whitespace normalization regex pattern is valid and should compile")
+});
+
+/// Normalizes multiple consecutive newlines
+pub(crate) static NEWLINE_NORMALIZE: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"\n\s*\n\s*\n+").expect("Newline normalization regex pattern is valid and should compile")
+});
+
+/// Cleans up newline sequences
+pub(crate) static NEWLINE_CLEANUP: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"\n+").expect("Newline cleanup regex pattern is valid and should compile"));

data/vendor/kreuzberg/src/utils/quality/scoring.rs

@@ -0,0 +1,178 @@
+//! Quality scoring functions
+//!
+//! This module provides functions to calculate quality scores and penalties
+//! based on various text characteristics.
+
+use super::patterns::*;
+use ahash::AHashMap;
+use regex::Regex;
+
+// ============================================================================
+// Scoring Constants and Weights
+// ============================================================================
+
+pub(crate) const OCR_PENALTY_WEIGHT: f64 = 0.3;
+pub(crate) const SCRIPT_PENALTY_WEIGHT: f64 = 0.2;
+pub(crate) const NAV_PENALTY_WEIGHT: f64 = 0.1;
+pub(crate) const STRUCTURE_BONUS_WEIGHT: f64 = 0.2;
+pub(crate) const METADATA_BONUS_WEIGHT: f64 = 0.1;
+
+pub(crate) const MIN_TEXT_LENGTH: usize = 10;
+pub(crate) const LARGE_TEXT_LENGTH: usize = 1000;
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+/// Sums the total length of all regex matches in the text
+#[inline]
+pub(crate) fn sum_match_lengths(text: &str, pattern: &Regex) -> usize {
+    pattern.find_iter(text).map(|m| m.len()).sum()
+}
+
+// ============================================================================
+// Penalty Calculation Functions
+// ============================================================================
+
+/// Calculate penalty based on OCR artifacts in the text
+#[inline]
+pub(crate) fn calculate_ocr_penalty(text: &str, total_chars: f64) -> f64 {
+    if total_chars == 0.0 {
+        return 0.0;
+    }
+
+    if !text.contains("  ") && !text.contains("...") {
+        return 0.0;
+    }
+
+    let artifact_chars = sum_match_lengths(text, &SCATTERED_CHARS_PATTERN)
+        + sum_match_lengths(text, &REPEATED_PUNCT_PATTERN)
+        + count_non_table_dash_artifacts(text)
+        + sum_match_lengths(text, &ISOLATED_PUNCT_PATTERN)
+        + sum_match_lengths(text, &MALFORMED_WORDS_PATTERN)
+        + sum_match_lengths(text, &EXCESSIVE_WHITESPACE_PATTERN);
+
+    (artifact_chars as f64 / total_chars).min(1.0)
+}
+
+/// Count dash artifacts while preserving table separators
+#[inline]
+pub(crate) fn count_non_table_dash_artifacts(text: &str) -> usize {
+    let mut artifact_count = 0;
+
+    for line in text.lines() {
+        let trimmed = line.trim();
+        let is_table_separator = trimmed.starts_with('|')
+            && trimmed.ends_with('|')
+            && trimmed
+                .chars()
+                .all(|c| c == '|' || c == '-' || c.is_whitespace() || c == ':');
+
+        if !is_table_separator {
+            for m in DASH_PATTERN.find_iter(line) {
+                artifact_count += m.len();
+            }
+        }
+    }
+
+    artifact_count
+}
+
+/// Calculate penalty based on embedded scripts and code
+#[inline]
+pub(crate) fn calculate_script_penalty(text: &str, total_chars: f64) -> f64 {
+    if total_chars == 0.0 {
+        return 0.0;
+    }
+
+    if !text.contains("function") && !text.contains("<script") && !text.contains("<style") {
+        return 0.0;
+    }
+
+    let script_chars = sum_match_lengths(text, &JS_FUNCTION_PATTERN)
+        + sum_match_lengths(text, &CSS_RULES_PATTERN)
+        + sum_match_lengths(text, &SCRIPT_TAG_PATTERN)
+        + sum_match_lengths(text, &STYLE_TAG_PATTERN);
+
+    (script_chars as f64 / total_chars).min(1.0)
+}
+
+/// Calculate penalty based on navigation elements
+#[inline]
+pub(crate) fn calculate_navigation_penalty(text: &str, total_chars: f64) -> f64 {
+    if total_chars == 0.0 {
+        return 0.0;
+    }
+
+    let nav_chars = sum_match_lengths(text, &NAV_WORDS_PATTERN)
+        + sum_match_lengths(text, &BREADCRUMB_PATTERN)
+        + sum_match_lengths(text, &PAGINATION_PATTERN);
+
+    (nav_chars as f64 / total_chars).min(1.0)
+}
+
+// ============================================================================
+// Bonus Calculation Functions
+// ============================================================================
+
+/// Calculate bonus based on document metadata quality
+#[inline]
+pub(crate) fn calculate_metadata_bonus(metadata: &AHashMap<String, String>) -> f64 {
+    const IMPORTANT_FIELDS: &[&str] = &["title", "author", "subject", "description", "keywords"];
+
+    let present_fields = IMPORTANT_FIELDS
+        .iter()
+        .filter(|&&field| metadata.contains_key(field))
+        .count();
+
+    present_fields as f64 / IMPORTANT_FIELDS.len() as f64
+}
+
+/// Compute a heuristic score (0.0–1.0) describing how clean the extracted text is.
+///
+/// The scoring pipeline rewards well-structured prose while penalising OCR artefacts,
+/// embedded scripts, and navigation chrome. Supplying document metadata allows the
+/// function to include contextual bonuses.
+///
+/// ```rust
+/// use ahash::AHashMap;
+/// use kreuzberg::utils::quality::calculate_quality_score;
+///
+/// let text = "Executive Summary\n===================\nKreuzberg extracts documents quickly.";
+/// let score = calculate_quality_score(text, None);
+/// assert!(score > 0.7);
+/// ```
+pub fn calculate_quality_score(text: &str, metadata: Option<&AHashMap<String, String>>) -> f64 {
+    if text.is_empty() || text.trim().is_empty() {
+        return 0.0;
+    }
+
+    let total_chars = text.len() as f64;
+
+    if text.len() < MIN_TEXT_LENGTH {
+        return 0.1;
+    }
+
+    let mut score = 1.0;
+
+    if text.len() > LARGE_TEXT_LENGTH {
+        let ocr_penalty = calculate_ocr_penalty(text, total_chars);
+        let script_penalty = calculate_script_penalty(text, total_chars);
+        let nav_penalty = calculate_navigation_penalty(text, total_chars);
+        let structure_bonus = super::heuristics::calculate_structure_bonus(text);
+
+        score -= ocr_penalty * OCR_PENALTY_WEIGHT;
+        score -= script_penalty * SCRIPT_PENALTY_WEIGHT;
+        score -= nav_penalty * NAV_PENALTY_WEIGHT;
+        score += structure_bonus * STRUCTURE_BONUS_WEIGHT;
+    } else {
+        score -= calculate_ocr_penalty(text, total_chars) * OCR_PENALTY_WEIGHT;
+        score += super::heuristics::calculate_structure_bonus(text) * STRUCTURE_BONUS_WEIGHT;
+    }
+
+    if let Some(metadata) = metadata {
+        score += calculate_metadata_bonus(metadata) * METADATA_BONUS_WEIGHT;
+    }
+
+    score.clamp(0.0, 1.0)
+}

data/vendor/kreuzberg/src/utils/string_pool/buffer_pool.rs

@@ -0,0 +1,325 @@
+//! Thread-safe reusable string buffer pool for reducing allocations.
+//!
+//! This module provides a pool of reusable String buffers that can be acquired,
+//! used, and automatically returned to the pool when dropped.
+
+use once_cell::sync::Lazy;
+use std::collections::VecDeque;
+use std::sync::Arc;
+
+#[cfg(feature = "pool-metrics")]
+use std::sync::atomic::AtomicUsize;
+
+#[cfg(feature = "pool-metrics")]
+use std::sync::atomic::Ordering;
+
+/// Configuration for the string buffer pool.
+pub struct PoolConfig {
+    /// Maximum buffers per size bucket
+    pub max_buffers_per_size: usize,
+    /// Initial capacity for new buffers
+    pub initial_capacity: usize,
+    /// Maximum capacity before discarding
+    pub max_capacity_before_discard: usize,
+}
+
+impl Default for PoolConfig {
+    fn default() -> Self {
+        Self {
+            max_buffers_per_size: 4,
+            initial_capacity: 4096,
+            max_capacity_before_discard: 65536,
+        }
+    }
+}
+
+/// Thread-safe reusable string buffer pool.
+///
+/// This pool allows allocation and reuse of String buffers to reduce memory allocations
+/// during document extraction. Buffers are returned to the pool with cleared contents
+/// but preserved capacity, ready for reuse.
+///
+/// # Thread Safety
+///
+/// The pool uses DashMap for lock-free concurrent access. Multiple threads can
+/// acquire and release buffers simultaneously.
+///
+/// # Usage
+///
+/// ```rust,ignore
+/// use kreuzberg::utils::string_pool::STRING_BUFFER_POOL;
+///
+/// // Acquire a buffer from the pool
+/// let mut buffer = STRING_BUFFER_POOL.acquire();
+/// buffer.push_str("some content");
+/// // Automatically returned to pool when dropped
+/// drop(buffer);
+/// ```
+pub struct StringBufferPool {
+    pool: dashmap::DashMap<usize, VecDeque<String>>,
+    config: PoolConfig,
+    #[cfg(feature = "pool-metrics")]
+    acquire_count: AtomicUsize,
+    #[cfg(feature = "pool-metrics")]
+    reuse_count: AtomicUsize,
+}
+
+impl StringBufferPool {
+    /// Create a new string buffer pool with given configuration.
+    pub fn new(config: PoolConfig) -> Self {
+        StringBufferPool {
+            pool: dashmap::DashMap::new(),
+            config,
+            #[cfg(feature = "pool-metrics")]
+            acquire_count: AtomicUsize::new(0),
+            #[cfg(feature = "pool-metrics")]
+            reuse_count: AtomicUsize::new(0),
+        }
+    }
+
+    /// Find the appropriate bucket size for a given capacity.
+    fn find_bucket(&self, capacity: usize) -> usize {
+        if capacity <= 1024 {
+            1024
+        } else if capacity <= 4096 {
+            4096
+        } else if capacity <= 16384 {
+            16384
+        } else if capacity <= 65536 {
+            65536
+        } else {
+            262144
+        }
+    }
+
+    /// Try to acquire a buffer from a specific bucket, returning it if found.
+    fn try_acquire_from_bucket(&self, bucket: usize) -> Option<String> {
+        if let Some(mut entry) = self.pool.get_mut(&bucket) {
+            entry.pop_front()
+        } else {
+            None
+        }
+    }
+
+    /// Acquire a string buffer from the pool, or allocate a new one if pool is exhausted.
+    ///
+    /// The returned buffer is automatically returned to the pool when dropped.
+    /// Must be called with the pool wrapped in Arc.
+    pub fn acquire(self: Arc<Self>) -> PooledString {
+        #[cfg(feature = "pool-metrics")]
+        self.acquire_count.fetch_add(1, Ordering::Relaxed);
+
+        let default_bucket = self.config.initial_capacity;
+        if let Some(buffer) = self.try_acquire_from_bucket(default_bucket) {
+            #[cfg(feature = "pool-metrics")]
+            self.reuse_count.fetch_add(1, Ordering::Relaxed);
+            return PooledString { buffer, pool: self };
+        }
+
+        for &bucket in &[1024, 16384, 65536] {
+            if let Some(buffer) = self.try_acquire_from_bucket(bucket) {
+                #[cfg(feature = "pool-metrics")]
+                self.reuse_count.fetch_add(1, Ordering::Relaxed);
+                return PooledString { buffer, pool: self };
+            }
+        }
+
+        PooledString {
+            buffer: String::with_capacity(self.config.initial_capacity),
+            pool: self,
+        }
+    }
+
+    /// Return a buffer to the pool for reuse.
+    pub fn release(&self, mut buffer: String) {
+        if buffer.capacity() > self.config.max_capacity_before_discard {
+            return;
+        }
+
+        let bucket = self.find_bucket(buffer.capacity());
+        buffer.clear();
+
+        if let Some(mut queue) = self.pool.get_mut(&bucket) {
+            if queue.len() < self.config.max_buffers_per_size {
+                queue.push_back(buffer);
+            }
+        } else {
+            let mut queue = VecDeque::with_capacity(self.config.max_buffers_per_size);
+            queue.push_back(buffer);
+            self.pool.insert(bucket, queue);
+        }
+    }
+
+    /// Get the current pool size across all buckets.
+    #[allow(dead_code)]
+    pub fn size(&self) -> usize {
+        self.pool.iter().map(|entry| entry.value().len()).sum()
+    }
+
+    /// Get buffer reuse metrics (only available with `pool-metrics` feature).
+    #[cfg(feature = "pool-metrics")]
+    pub fn metrics(&self) -> StringBufferPoolMetrics {
+        let acquire = self.acquire_count.load(Ordering::Relaxed);
+        let reuse = self.reuse_count.load(Ordering::Relaxed);
+        let hit_rate = if acquire == 0 {
+            0.0
+        } else {
+            (reuse as f64 / acquire as f64) * 100.0
+        };
+
+        StringBufferPoolMetrics {
+            total_acquires: acquire,
+            total_reuses: reuse,
+            hit_rate,
+        }
+    }
+}
+
+/// Metrics for StringBufferPool (only available with `pool-metrics` feature).
+#[cfg(feature = "pool-metrics")]
+#[derive(Debug, Clone, Copy)]
+pub struct StringBufferPoolMetrics {
+    /// Total number of acquire calls
+    pub total_acquires: usize,
+    /// Total number of buffer reuses from pool
+    pub total_reuses: usize,
+    /// Hit rate as percentage (0.0-100.0)
+    pub hit_rate: f64,
+}
+
+/// RAII wrapper for a pooled string buffer.
+///
+/// Automatically returns the buffer to the pool when dropped.
+pub struct PooledString {
+    buffer: String,
+    pool: Arc<StringBufferPool>,
+}
+
+impl PooledString {
+    /// Get mutable access to the underlying string buffer.
+    pub fn buffer_mut(&mut self) -> &mut String {
+        &mut self.buffer
+    }
+
+    /// Get immutable access to the underlying string buffer.
+    pub fn as_str(&self) -> &str {
+        self.buffer.as_str()
+    }
+}
+
+impl std::ops::Deref for PooledString {
+    type Target = String;
+
+    fn deref(&self) -> &Self::Target {
+        &self.buffer
+    }
+}
+
+impl std::ops::DerefMut for PooledString {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.buffer
+    }
+}
+
+impl Drop for PooledString {
+    fn drop(&mut self) {
+        let buffer = std::mem::take(&mut self.buffer);
+        self.pool.release(buffer);
+    }
+}
+
+impl std::fmt::Display for PooledString {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.buffer)
+    }
+}
+
+impl std::fmt::Debug for PooledString {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("PooledString").field(&self.buffer).finish()
+    }
+}
+
+/// Global string buffer pool for temporary allocations during extraction.
+pub static STRING_BUFFER_POOL: Lazy<Arc<StringBufferPool>> =
+    Lazy::new(|| Arc::new(StringBufferPool::new(PoolConfig::default())));
+
+/// Acquire a string buffer from the global pool.
+///
+/// The returned buffer is automatically returned to the pool when dropped.
+///
+/// # Example
+///
+/// ```rust,ignore
+/// let mut buffer = acquire_string_buffer();
+/// buffer.push_str("content");
+/// // Automatically returned to pool when buffer goes out of scope
+/// ```
+pub fn acquire_string_buffer() -> PooledString {
+    Arc::clone(&*STRING_BUFFER_POOL).acquire()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_buffer_pool_acquire_and_release() {
+        let config = PoolConfig::default();
+        let pool = Arc::new(StringBufferPool::new(config));
+
+        let mut buffer = pool.clone().acquire();
+        buffer.push_str("test content");
+        let capacity = buffer.capacity();
+
+        drop(buffer);
+
+        let buffer2 = pool.clone().acquire();
+        assert_eq!(buffer2.capacity(), capacity);
+        assert!(buffer2.is_empty());
+    }
+
+    #[test]
+    fn test_buffer_pool_size() {
+        let config = PoolConfig::default();
+        let pool = Arc::new(StringBufferPool::new(config));
+
+        assert_eq!(pool.size(), 0);
+
+        let buffer1 = pool.clone().acquire();
+        drop(buffer1);
+        assert_eq!(pool.size(), 1);
+
+        let buffer2 = pool.clone().acquire();
+        drop(buffer2);
+        assert_eq!(pool.size(), 1);
+    }
+
+    #[test]
+    fn test_buffer_pool_global() {
+        let buffer1 = acquire_string_buffer();
+        drop(buffer1);
+
+        let buffer2 = acquire_string_buffer();
+        assert!(buffer2.capacity() >= 4096);
+    }
+
+    #[test]
+    fn test_pooled_string_deref() {
+        let mut buffer = acquire_string_buffer();
+        buffer.push_str("hello");
+
+        assert_eq!(&*buffer, "hello");
+        assert_eq!(buffer.as_str(), "hello");
+        assert!(!buffer.is_empty());
+    }
+
+    #[test]
+    fn test_pooled_string_deref_mut() {
+        let mut buffer = acquire_string_buffer();
+        buffer.push_str("test");
+
+        buffer.buffer_mut().push_str(" more");
+        assert_eq!(buffer.as_str(), "test more");
+    }
+}