kreuzberg 4.0.0.rc1

data/vendor/kreuzberg/src/image/resize.rs

@@ -0,0 +1,89 @@
+use crate::error::{KreuzbergError, Result};
+use fast_image_resize::{FilterType, PixelType, ResizeAlg, ResizeOptions, Resizer, images::Image as FirImage};
+use image::{DynamicImage, ImageBuffer, Rgb};
+
+/// Resize an image using fast_image_resize with appropriate algorithm based on scale factor
+pub fn resize_image(image: &DynamicImage, new_width: u32, new_height: u32, scale_factor: f64) -> Result<DynamicImage> {
+    let rgb_image = image.to_rgb8();
+    let (width, height) = rgb_image.dimensions();
+
+    let src_image = FirImage::from_vec_u8(width, height, rgb_image.into_raw(), PixelType::U8x3)
+        .map_err(|e| KreuzbergError::parsing(format!("Failed to create source image: {e:?}")))?;
+
+    let mut dst_image = FirImage::new(new_width, new_height, PixelType::U8x3);
+
+    let algorithm = if scale_factor < 1.0 {
+        ResizeAlg::Convolution(FilterType::Lanczos3)
+    } else {
+        ResizeAlg::Convolution(FilterType::CatmullRom)
+    };
+
+    let mut resizer = Resizer::new();
+    resizer
+        .resize(&src_image, &mut dst_image, &ResizeOptions::new().resize_alg(algorithm))
+        .map_err(|e| KreuzbergError::parsing(format!("Resize failed: {e:?}")))?;
+
+    let buffer = dst_image.into_vec();
+    let img_buffer = ImageBuffer::<Rgb<u8>, Vec<u8>>::from_raw(new_width, new_height, buffer)
+        .ok_or_else(|| KreuzbergError::parsing("Failed to create image buffer".to_string()))?;
+
+    Ok(DynamicImage::ImageRgb8(img_buffer))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use image::Rgb;
+
+    fn create_test_image() -> DynamicImage {
+        let mut img = ImageBuffer::new(100, 100);
+        for y in 0..100 {
+            for x in 0..100 {
+                img.put_pixel(x, y, Rgb([255u8, 0u8, 0u8]));
+            }
+        }
+        DynamicImage::ImageRgb8(img)
+    }
+
+    #[test]
+    fn test_resize_image_downscale() {
+        let img = create_test_image();
+        let result = resize_image(&img, 50, 50, 0.5);
+        assert!(result.is_ok());
+        let resized = result.unwrap();
+        assert_eq!(resized.width(), 50);
+        assert_eq!(resized.height(), 50);
+    }
+
+    #[test]
+    fn test_resize_image_upscale() {
+        let img = create_test_image();
+        let result = resize_image(&img, 200, 200, 2.0);
+        assert!(result.is_ok());
+        let resized = result.unwrap();
+        assert_eq!(resized.width(), 200);
+        assert_eq!(resized.height(), 200);
+    }
+
+    #[test]
+    fn test_resize_image_no_scale() {
+        let img = create_test_image();
+        let result = resize_image(&img, 100, 100, 1.0);
+        assert!(result.is_ok());
+        let resized = result.unwrap();
+        assert_eq!(resized.width(), 100);
+        assert_eq!(resized.height(), 100);
+    }
+
+    #[test]
+    fn test_resize_preserves_aspect_ratio() {
+        let img = create_test_image();
+        let result = resize_image(&img, 50, 50, 0.5);
+        assert!(result.is_ok());
+        let resized = result.unwrap();
+
+        let original_aspect = img.width() as f64 / img.height() as f64;
+        let resized_aspect = resized.width() as f64 / resized.height() as f64;
+        assert!((original_aspect - resized_aspect).abs() < 0.01);
+    }
+}

data/vendor/kreuzberg/src/keywords/config.rs

@@ -0,0 +1,154 @@
+//! Configuration for keyword extraction.
+
+use super::types::KeywordAlgorithm;
+use serde::{Deserialize, Serialize};
+
+/// YAKE-specific parameters.
+#[cfg(feature = "keywords-yake")]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct YakeParams {
+    /// Window size for co-occurrence analysis (default: 2).
+    ///
+    /// Controls the context window for computing co-occurrence statistics.
+    pub window_size: usize,
+}
+
+#[cfg(feature = "keywords-yake")]
+impl Default for YakeParams {
+    fn default() -> Self {
+        Self { window_size: 2 }
+    }
+}
+
+/// RAKE-specific parameters.
+#[cfg(feature = "keywords-rake")]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RakeParams {
+    /// Minimum word length to consider (default: 1).
+    pub min_word_length: usize,
+
+    /// Maximum words in a keyword phrase (default: 3).
+    pub max_words_per_phrase: usize,
+}
+
+#[cfg(feature = "keywords-rake")]
+impl Default for RakeParams {
+    fn default() -> Self {
+        Self {
+            min_word_length: 1,
+            max_words_per_phrase: 3,
+        }
+    }
+}
+
+/// Keyword extraction configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct KeywordConfig {
+    /// Algorithm to use for extraction.
+    pub algorithm: KeywordAlgorithm,
+
+    /// Maximum number of keywords to extract (default: 10).
+    pub max_keywords: usize,
+
+    /// Minimum score threshold (0.0-1.0, default: 0.0).
+    ///
+    /// Keywords with scores below this threshold are filtered out.
+    /// Note: Score ranges differ between algorithms.
+    pub min_score: f32,
+
+    /// N-gram range for keyword extraction (min, max).
+    ///
+    /// (1, 1) = unigrams only
+    /// (1, 2) = unigrams and bigrams
+    /// (1, 3) = unigrams, bigrams, and trigrams (default)
+    pub ngram_range: (usize, usize),
+
+    /// Language code for stopword filtering (e.g., "en", "de", "fr").
+    ///
+    /// If None, no stopword filtering is applied.
+    pub language: Option<String>,
+
+    /// YAKE-specific tuning parameters.
+    #[cfg(feature = "keywords-yake")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub yake_params: Option<YakeParams>,
+
+    /// RAKE-specific tuning parameters.
+    #[cfg(feature = "keywords-rake")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rake_params: Option<RakeParams>,
+}
+
+impl Default for KeywordConfig {
+    fn default() -> Self {
+        Self {
+            algorithm: KeywordAlgorithm::default(),
+            max_keywords: 10,
+            min_score: 0.0,
+            ngram_range: (1, 3),
+            language: Some("en".to_string()),
+            #[cfg(feature = "keywords-yake")]
+            yake_params: None,
+            #[cfg(feature = "keywords-rake")]
+            rake_params: None,
+        }
+    }
+}
+
+impl KeywordConfig {
+    /// Create a new configuration with YAKE algorithm.
+    #[cfg(feature = "keywords-yake")]
+    pub fn yake() -> Self {
+        Self {
+            algorithm: KeywordAlgorithm::Yake,
+            ..Default::default()
+        }
+    }
+
+    /// Create a new configuration with RAKE algorithm.
+    #[cfg(feature = "keywords-rake")]
+    pub fn rake() -> Self {
+        Self {
+            algorithm: KeywordAlgorithm::Rake,
+            ..Default::default()
+        }
+    }
+
+    /// Set maximum number of keywords to extract.
+    pub fn with_max_keywords(mut self, max: usize) -> Self {
+        self.max_keywords = max;
+        self
+    }
+
+    /// Set minimum score threshold.
+    pub fn with_min_score(mut self, score: f32) -> Self {
+        self.min_score = score;
+        self
+    }
+
+    /// Set n-gram range.
+    pub fn with_ngram_range(mut self, min: usize, max: usize) -> Self {
+        self.ngram_range = (min, max);
+        self
+    }
+
+    /// Set language for stopword filtering.
+    pub fn with_language(mut self, lang: impl Into<String>) -> Self {
+        self.language = Some(lang.into());
+        self
+    }
+
+    /// Set YAKE-specific parameters.
+    #[cfg(feature = "keywords-yake")]
+    pub fn with_yake_params(mut self, params: YakeParams) -> Self {
+        self.yake_params = Some(params);
+        self
+    }
+
+    /// Set RAKE-specific parameters.
+    #[cfg(feature = "keywords-rake")]
+    pub fn with_rake_params(mut self, params: RakeParams) -> Self {
+        self.rake_params = Some(params);
+        self
+    }
+}

data/vendor/kreuzberg/src/keywords/mod.rs

@@ -0,0 +1,237 @@
+//! Keyword extraction module.
+//!
+//! Provides unified keyword extraction interface supporting multiple algorithms:
+//! - YAKE (Yet Another Keyword Extractor) - statistical approach
+//! - RAKE (Rapid Automatic Keyword Extraction) - co-occurrence based
+//!
+//! # Feature Flags
+//!
+//! - `keywords-yake`: Enable YAKE algorithm
+//! - `keywords-rake`: Enable RAKE algorithm
+//! - `keywords`: Enable both algorithms (default in `full` feature)
+//!
+//! # Examples
+//!
+//! ```rust,no_run
+//! # use kreuzberg::keywords::{extract_keywords, KeywordConfig};
+//! let text = "Rust is a systems programming language focused on safety and performance.";
+//!
+//! // Use default algorithm (YAKE if available)
+//! let config = KeywordConfig::default();
+//! let keywords = extract_keywords(text, &config).unwrap();
+//!
+//! for keyword in keywords {
+//!     println!("{}: {:.3}", keyword.text, keyword.score);
+//! }
+//! ```
+//!
+//! ```rust,no_run
+//! # #[cfg(feature = "keywords-rake")]
+//! # {
+//! # use kreuzberg::keywords::{extract_keywords, KeywordConfig};
+//! // Use RAKE algorithm explicitly
+//! let text = "Machine learning models require large datasets.";
+//! let config = KeywordConfig::rake()
+//!     .with_max_keywords(5)
+//!     .with_min_score(0.3);
+//!
+//! let keywords = extract_keywords(text, &config).unwrap();
+//! # }
+//! ```
+
+use crate::Result;
+use crate::plugins::registry::get_post_processor_registry;
+use once_cell::sync::Lazy;
+use std::sync::Arc;
+
+pub mod config;
+pub mod processor;
+pub mod types;
+
+#[cfg(feature = "keywords-yake")]
+mod yake;
+
+#[cfg(feature = "keywords-rake")]
+mod rake;
+
+pub use config::KeywordConfig;
+pub use processor::KeywordExtractor;
+
+#[cfg(feature = "keywords-rake")]
+pub use config::RakeParams;
+
+#[cfg(feature = "keywords-yake")]
+pub use config::YakeParams;
+pub use types::{Keyword, KeywordAlgorithm};
+
+/// Extract keywords from text using the specified algorithm.
+///
+/// This is the unified entry point for keyword extraction. The algorithm
+/// used is determined by `config.algorithm`.
+///
+/// # Arguments
+///
+/// * `text` - The text to extract keywords from
+/// * `config` - Keyword extraction configuration
+///
+/// # Returns
+///
+/// A vector of keywords sorted by relevance (highest score first).
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - The specified algorithm feature is not enabled
+/// - Keyword extraction fails
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// # use kreuzberg::keywords::{extract_keywords, KeywordConfig};
+/// let text = "Document intelligence with Rust provides memory safety.";
+/// let config = KeywordConfig::default()
+///     .with_max_keywords(10)
+///     .with_language("en");
+///
+/// let keywords = extract_keywords(text, &config)?;
+///
+/// for keyword in keywords {
+///     println!("{}: {:.3}", keyword.text, keyword.score);
+/// }
+/// # Ok::<(), kreuzberg::KreuzbergError>(())
+/// ```
+pub fn extract_keywords(text: &str, config: &KeywordConfig) -> Result<Vec<Keyword>> {
+    match config.algorithm {
+        #[cfg(feature = "keywords-yake")]
+        KeywordAlgorithm::Yake => yake::extract_keywords_yake(text, config),
+
+        #[cfg(feature = "keywords-rake")]
+        KeywordAlgorithm::Rake => rake::extract_keywords_rake(text, config),
+
+        #[cfg(not(any(feature = "keywords-yake", feature = "keywords-rake")))]
+        _ => Err(crate::KreuzbergError::Other(
+            "No keyword extraction algorithm feature enabled".to_string(),
+        )),
+    }
+}
+
+/// Lazy-initialized flag that ensures keyword processor is registered exactly once.
+///
+/// This static is accessed on first use to automatically register the
+/// keyword extraction processor with the plugin registry.
+static PROCESSOR_INITIALIZED: Lazy<Result<()>> = Lazy::new(register_keyword_processor);
+
+/// Ensure the keyword processor is registered.
+///
+/// This function is called automatically when needed.
+/// It's safe to call multiple times - registration only happens once.
+pub fn ensure_initialized() -> Result<()> {
+    PROCESSOR_INITIALIZED
+        .as_ref()
+        .map(|_| ())
+        .map_err(|e| crate::KreuzbergError::Plugin {
+            message: format!("Failed to register keyword processor: {}", e),
+            plugin_name: "keyword-extraction".to_string(),
+        })
+}
+
+/// Register the keyword extraction processor with the global registry.
+///
+/// This function should be called once at application startup to register
+/// the keyword extraction post-processor.
+///
+/// **Note:** This is called automatically on first use.
+/// Explicit calling is optional.
+///
+/// # Example
+///
+/// ```rust
+/// # #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+/// use kreuzberg::keywords::register_keyword_processor;
+///
+/// # #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+/// # fn main() -> kreuzberg::Result<()> {
+/// register_keyword_processor()?;
+/// # Ok(())
+/// # }
+/// # #[cfg(not(any(feature = "keywords-yake", feature = "keywords-rake")))]
+/// # fn main() {}
+/// ```
+pub fn register_keyword_processor() -> Result<()> {
+    let registry = get_post_processor_registry();
+    let mut registry = registry
+        .write()
+        .map_err(|e| crate::KreuzbergError::Other(format!("Post-processor registry lock poisoned: {}", e)))?;
+
+    registry.register(Arc::new(KeywordExtractor), 50)?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_extract_keywords_default_algorithm() {
+        let text = "Rust programming language provides memory safety and performance.";
+        let config = KeywordConfig::default();
+
+        let keywords = extract_keywords(text, &config).unwrap();
+
+        assert!(!keywords.is_empty(), "Should extract keywords");
+        assert!(keywords.len() <= config.max_keywords);
+    }
+
+    #[cfg(feature = "keywords-yake")]
+    #[test]
+    fn test_extract_keywords_yake() {
+        let text = "Natural language processing using Rust is efficient and safe.";
+        let config = KeywordConfig::yake();
+
+        let keywords = extract_keywords(text, &config).unwrap();
+
+        assert!(!keywords.is_empty());
+        assert_eq!(keywords[0].algorithm, KeywordAlgorithm::Yake);
+    }
+
+    #[cfg(feature = "keywords-rake")]
+    #[test]
+    fn test_extract_keywords_rake() {
+        let text = "Natural language processing using Rust is efficient and safe.";
+        let config = KeywordConfig::rake();
+
+        let keywords = extract_keywords(text, &config).unwrap();
+
+        assert!(!keywords.is_empty());
+        assert_eq!(keywords[0].algorithm, KeywordAlgorithm::Rake);
+    }
+
+    #[cfg(all(feature = "keywords-yake", feature = "keywords-rake"))]
+    #[test]
+    fn test_compare_algorithms() {
+        let text = "Machine learning and artificial intelligence are transforming technology. \
+                    Deep learning models require substantial computational resources.";
+
+        let yake_config = KeywordConfig::yake().with_max_keywords(5);
+        let yake_keywords = extract_keywords(text, &yake_config).unwrap();
+
+        let rake_config = KeywordConfig::rake().with_max_keywords(5);
+        let rake_keywords = extract_keywords(text, &rake_config).unwrap();
+
+        assert!(!yake_keywords.is_empty());
+        assert!(!rake_keywords.is_empty());
+
+        assert!(yake_keywords.iter().all(|k| k.algorithm == KeywordAlgorithm::Yake));
+        assert!(rake_keywords.iter().all(|k| k.algorithm == KeywordAlgorithm::Rake));
+
+        println!(
+            "YAKE keywords: {:?}",
+            yake_keywords.iter().map(|k| &k.text).collect::<Vec<_>>()
+        );
+        println!(
+            "RAKE keywords: {:?}",
+            rake_keywords.iter().map(|k| &k.text).collect::<Vec<_>>()
+        );
+    }
+}