spellkit 0.1.0.pre.1

data/ext/spellkit/src/lib.rs

@@ -0,0 +1,255 @@
+mod symspell;
+mod guards;
+
+use magnus::{class, define_module, function, method, prelude::*, Error, RArray, RHash, Ruby, Value, TryConvert};
+use std::sync::{Arc, RwLock};
+use symspell::SymSpell;
+use guards::Guards;
+
+use std::time::{SystemTime, UNIX_EPOCH};
+
+#[derive(Clone)]
+#[magnus::wrap(class = "SpellKit::Checker", free_immediately, size)]
+struct Checker {
+    state: Arc<RwLock<CheckerState>>,
+}
+
+struct CheckerState {
+    symspell: Option<SymSpell>,
+    guards: Guards,
+    loaded: bool,
+    frequency_threshold: f64,
+    loaded_at: Option<u64>,
+    dictionary_size: usize,
+    edit_distance: usize,
+}
+
+impl CheckerState {
+    fn new() -> Self {
+        Self {
+            symspell: None,
+            guards: Guards::new(),
+            loaded: false,
+            frequency_threshold: 10.0,
+            loaded_at: None,
+            dictionary_size: 0,
+            edit_distance: 1,
+        }
+    }
+}
+
+impl Checker {
+    fn new() -> Self {
+        Self {
+            state: Arc::new(RwLock::new(CheckerState::new())),
+        }
+    }
+
+    fn load_full(&self, config: RHash) -> Result<(), Error> {
+        let ruby = Ruby::get().unwrap();
+
+        // Required: dictionary path
+        let dictionary_path: String = TryConvert::try_convert(
+            config.fetch::<_, Value>("dictionary_path")
+                .map_err(|_| Error::new(ruby.exception_arg_error(), "dictionary_path is required"))?
+        )?;
+
+        let content = std::fs::read_to_string(&dictionary_path)
+            .map_err(|e| Error::new(ruby.exception_runtime_error(), format!("Failed to read dictionary file: {}", e)))?;
+
+    // Optional: edit distance
+    let edit_dist: usize = config.get("edit_distance")
+        .and_then(|v: Value| TryConvert::try_convert(v).ok())
+        .unwrap_or(1);
+
+    if edit_dist > 2 {
+        return Err(Error::new(ruby.exception_arg_error(), "edit_distance must be 1 or 2"));
+    }
+
+    let mut words = Vec::new();
+    for line in content.lines() {
+        let parts: Vec<&str> = line.split_whitespace().collect();
+        if parts.len() == 2 {
+            if let Ok(freq) = parts[1].parse::<u64>() {
+                words.push((parts[0].to_string(), freq));
+            }
+        }
+    }
+
+    let dictionary_size = words.len();
+    let mut symspell = SymSpell::new(edit_dist);
+    symspell.load_dictionary(words);
+
+    let mut guards = Guards::new();
+
+    // Load optional protected terms file
+    if let Some(protected_path) = config.get("protected_path") {
+        let path: String = TryConvert::try_convert(protected_path)?;
+        if let Ok(content) = std::fs::read_to_string(path) {
+            guards.load_protected(&content);
+        }
+    }
+
+    // Load optional protected patterns
+    if let Some(patterns_value) = config.get("protected_patterns") {
+        let patterns: RArray = TryConvert::try_convert(patterns_value)?;
+        for pattern_value in patterns.into_iter() {
+            let pattern: String = TryConvert::try_convert(pattern_value)?;
+            guards.add_pattern(&pattern)
+                .map_err(|e| Error::new(ruby.exception_arg_error(), e))?;
+        }
+    }
+
+    // Optional frequency threshold
+    let frequency_threshold: f64 = config.get("frequency_threshold")
+        .and_then(|v: Value| TryConvert::try_convert(v).ok())
+        .unwrap_or(10.0);
+
+        let loaded_at = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .ok()
+            .map(|d| d.as_secs());
+
+        let mut state = self.state.write().unwrap();
+        state.symspell = Some(symspell);
+        state.guards = guards;
+        state.frequency_threshold = frequency_threshold;
+        state.loaded = true;
+        state.loaded_at = loaded_at;
+        state.dictionary_size = dictionary_size;
+        state.edit_distance = edit_dist;
+
+        Ok(())
+    }
+
+    fn suggest(&self, word: String, max: Option<usize>) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let max_suggestions = max.unwrap_or(5);
+        let state = self.state.read().unwrap();
+
+        if !state.loaded {
+            return Err(Error::new(ruby.exception_runtime_error(), "Dictionary not loaded. Call load! first"));
+        }
+
+        if let Some(ref symspell) = state.symspell {
+            let suggestions = symspell.suggest(&word, max_suggestions);
+            let result = RArray::new();
+
+            for suggestion in suggestions {
+                let hash = RHash::new();
+                hash.aset("term", suggestion.term)?;
+                hash.aset("distance", suggestion.distance)?;
+                hash.aset("freq", suggestion.frequency)?;
+                result.push(hash)?;
+            }
+
+            Ok(result)
+        } else {
+            Err(Error::new(ruby.exception_runtime_error(), "SymSpell not initialized"))
+        }
+    }
+
+    fn correct_if_unknown(&self, word: String, use_guard: Option<bool>) -> Result<String, Error> {
+        let ruby = Ruby::get().unwrap();
+        let state = self.state.read().unwrap();
+
+        if !state.loaded {
+            return Err(Error::new(ruby.exception_runtime_error(), "Dictionary not loaded. Call load! first"));
+        }
+
+        // Check if word is protected
+        if use_guard.unwrap_or(false) {
+            let normalized = SymSpell::normalize_word(&word);
+            if state.guards.is_protected_normalized(&word, &normalized) {
+                return Ok(word);
+            }
+        }
+
+        if let Some(ref symspell) = state.symspell {
+            let suggestions = symspell.suggest(&word, 5);
+
+            // If exact match exists, return original
+            if !suggestions.is_empty() && suggestions[0].distance == 0 {
+                return Ok(word);
+            }
+
+            // Find best correction with frequency gating
+            for suggestion in &suggestions {
+                if suggestion.distance <= 1 {
+                    // Check frequency threshold - correction should be significantly more common
+                    // Since we don't have the original word's frequency, we'll just take any ED=1 match
+                    // In a full implementation, we'd check if suggestion.frequency >= threshold * original_freq
+                    return Ok(suggestion.term.clone());
+                }
+            }
+
+            Ok(word)
+        } else {
+            Err(Error::new(ruby.exception_runtime_error(), "SymSpell not initialized"))
+        }
+    }
+
+    fn correct_tokens(&self, tokens: RArray, use_guard: Option<bool>) -> Result<RArray, Error> {
+        let result = RArray::new();
+        let guard = use_guard.unwrap_or(false);
+
+        for token in tokens.into_iter() {
+            let word: String = TryConvert::try_convert(token)?;
+            let corrected = self.correct_if_unknown(word, Some(guard))?;
+            result.push(corrected)?;
+        }
+
+        Ok(result)
+    }
+
+    fn stats(&self) -> Result<RHash, Error> {
+        let state = self.state.read().unwrap();
+        let stats = RHash::new();
+
+        if !state.loaded {
+            stats.aset("loaded", false)?;
+            return Ok(stats);
+        }
+
+        stats.aset("loaded", true)?;
+        stats.aset("dictionary_size", state.dictionary_size)?;
+        stats.aset("edit_distance", state.edit_distance)?;
+
+        if let Some(loaded_at) = state.loaded_at {
+            stats.aset("loaded_at", loaded_at)?;
+        }
+
+        Ok(stats)
+    }
+
+    fn healthcheck(&self) -> Result<(), Error> {
+        let ruby = Ruby::get().unwrap();
+        let state = self.state.read().unwrap();
+
+        if !state.loaded {
+            return Err(Error::new(ruby.exception_runtime_error(), "Dictionary not loaded"));
+        }
+
+        if state.symspell.is_none() {
+            return Err(Error::new(ruby.exception_runtime_error(), "SymSpell not initialized"));
+        }
+
+        Ok(())
+    }
+}
+
+#[magnus::init]
+fn init(_ruby: &Ruby) -> Result<(), Error> {
+    let module = define_module("SpellKit")?;
+    let checker_class = module.define_class("Checker", class::object())?;
+
+    checker_class.define_singleton_method("new", function!(Checker::new, 0))?;
+    checker_class.define_method("load!", method!(Checker::load_full, 1))?;
+    checker_class.define_method("suggest", method!(Checker::suggest, 2))?;
+    checker_class.define_method("correct_if_unknown", method!(Checker::correct_if_unknown, 2))?;
+    checker_class.define_method("correct_tokens", method!(Checker::correct_tokens, 2))?;
+    checker_class.define_method("stats", method!(Checker::stats, 0))?;
+    checker_class.define_method("healthcheck", method!(Checker::healthcheck, 0))?;
+
+    Ok(())
+}

data/ext/spellkit/src/symspell.rs

@@ -0,0 +1,264 @@
+use hashbrown::{HashMap, HashSet};
+use std::cmp::Ordering;
+use unicode_normalization::UnicodeNormalization;
+
+#[derive(Debug, Clone)]
+pub struct Suggestion {
+    pub term: String,
+    pub distance: usize,
+    pub frequency: u64,
+}
+
+impl Suggestion {
+    pub fn new(term: String, distance: usize, frequency: u64) -> Self {
+        Self {
+            term,
+            distance,
+            frequency,
+        }
+    }
+}
+
+impl Ord for Suggestion {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.distance
+            .cmp(&other.distance)
+            .then_with(|| other.frequency.cmp(&self.frequency))
+            .then_with(|| self.term.cmp(&other.term))
+    }
+}
+
+impl PartialOrd for Suggestion {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl PartialEq for Suggestion {
+    fn eq(&self, other: &Self) -> bool {
+        self.term == other.term && self.distance == other.distance && self.frequency == other.frequency
+    }
+}
+
+impl Eq for Suggestion {}
+
+pub struct SymSpell {
+    deletes: HashMap<String, HashSet<String>>,
+    words: HashMap<String, u64>,
+    max_edit_distance: usize,
+}
+
+impl SymSpell {
+    pub fn new(max_edit_distance: usize) -> Self {
+        Self {
+            deletes: HashMap::new(),
+            words: HashMap::new(),
+            max_edit_distance,
+        }
+    }
+
+    pub fn normalize_word(word: &str) -> String {
+        word.nfkd()
+            .filter(|c| !c.is_control() && !c.is_whitespace())
+            .collect::<String>()
+            .to_lowercase()
+    }
+
+    pub fn load_dictionary(&mut self, words: Vec<(String, u64)>) {
+        for (word, freq) in words {
+            let normalized = Self::normalize_word(&word);
+            self.add_word(&normalized, freq);
+        }
+    }
+
+    pub fn add_word(&mut self, word: &str, frequency: u64) {
+        self.words.insert(word.to_string(), frequency);
+
+        let deletes = self.get_deletes(word, self.max_edit_distance);
+        for delete in deletes {
+            self.deletes
+                .entry(delete)
+                .or_insert_with(HashSet::new)
+                .insert(word.to_string());
+        }
+    }
+
+    fn get_deletes(&self, word: &str, edit_distance: usize) -> HashSet<String> {
+        let mut deletes = HashSet::new();
+        if edit_distance == 0 {
+            return deletes;
+        }
+
+        let mut queue = vec![word.to_string()];
+        let mut processed = HashSet::new();
+
+        for _ in 0..edit_distance {
+            let mut temp_queue = Vec::new();
+            for item in queue {
+                if processed.contains(&item) {
+                    continue;
+                }
+                processed.insert(item.clone());
+
+                for delete in self.generate_deletes(&item) {
+                    if delete.len() >= 1 {
+                        deletes.insert(delete.clone());
+                        temp_queue.push(delete);
+                    }
+                }
+            }
+            queue = temp_queue;
+        }
+
+        deletes
+    }
+
+    fn generate_deletes(&self, word: &str) -> Vec<String> {
+        let chars: Vec<char> = word.chars().collect();
+        let mut deletes = Vec::new();
+
+        for i in 0..chars.len() {
+            let mut new_word = String::new();
+            for (j, &ch) in chars.iter().enumerate() {
+                if j != i {
+                    new_word.push(ch);
+                }
+            }
+            deletes.push(new_word);
+        }
+
+        deletes
+    }
+
+    pub fn suggest(&self, word: &str, max_suggestions: usize) -> Vec<Suggestion> {
+        let normalized = Self::normalize_word(word);
+        let mut suggestions = Vec::new();
+        let mut seen = HashSet::new();
+
+        if let Some(&freq) = self.words.get(&normalized) {
+            suggestions.push(Suggestion::new(normalized.clone(), 0, freq));
+            seen.insert(normalized.clone());
+        }
+
+        let input_deletes = self.get_deletes(&normalized, self.max_edit_distance);
+
+        for delete in &input_deletes {
+            // Check if this delete is itself a dictionary word (important for finding words shorter than input)
+            if !seen.contains(delete) {
+                if let Some(&freq) = self.words.get(delete) {
+                    let distance = self.edit_distance(&normalized, delete);
+                    if distance <= self.max_edit_distance {
+                        suggestions.push(Suggestion::new(delete.clone(), distance, freq));
+                        seen.insert(delete.clone());
+                    }
+                }
+            }
+
+            // Check the deletes map for candidates
+            if let Some(candidates) = self.deletes.get(delete) {
+                for candidate in candidates {
+                    if seen.contains(candidate) {
+                        continue;
+                    }
+
+                    let distance = self.edit_distance(&normalized, candidate);
+                    if distance <= self.max_edit_distance {
+                        if let Some(&freq) = self.words.get(candidate) {
+                            suggestions.push(Suggestion::new(candidate.clone(), distance, freq));
+                            seen.insert(candidate.clone());
+                        }
+                    }
+                }
+            }
+        }
+
+        if let Some(candidates) = self.deletes.get(&normalized) {
+            for candidate in candidates {
+                if seen.contains(candidate) {
+                    continue;
+                }
+
+                let distance = self.edit_distance(&normalized, candidate);
+                if distance <= self.max_edit_distance {
+                    if let Some(&freq) = self.words.get(candidate) {
+                        suggestions.push(Suggestion::new(candidate.clone(), distance, freq));
+                        seen.insert(candidate.clone());
+                    }
+                }
+            }
+        }
+
+        suggestions.sort();
+        suggestions.truncate(max_suggestions);
+        suggestions
+    }
+
+    fn edit_distance(&self, s1: &str, s2: &str) -> usize {
+        let len1 = s1.chars().count();
+        let len2 = s2.chars().count();
+
+        if len1 == 0 {
+            return len2;
+        }
+        if len2 == 0 {
+            return len1;
+        }
+
+        let s1_chars: Vec<char> = s1.chars().collect();
+        let s2_chars: Vec<char> = s2.chars().collect();
+
+        let mut prev_row: Vec<usize> = (0..=len2).collect();
+        let mut curr_row = vec![0; len2 + 1];
+
+        for i in 1..=len1 {
+            curr_row[0] = i;
+
+            for j in 1..=len2 {
+                let cost = if s1_chars[i - 1] == s2_chars[j - 1] {
+                    0
+                } else {
+                    1
+                };
+
+                curr_row[j] = std::cmp::min(
+                    std::cmp::min(
+                        prev_row[j] + 1,      // deletion
+                        curr_row[j - 1] + 1   // insertion
+                    ),
+                    prev_row[j - 1] + cost    // substitution
+                );
+            }
+
+            std::mem::swap(&mut prev_row, &mut curr_row);
+        }
+
+        prev_row[len2]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_edit_distance() {
+        let symspell = SymSpell::new(2);
+        assert_eq!(symspell.edit_distance("test", "test"), 0);
+        assert_eq!(symspell.edit_distance("test", "tests"), 1);
+        assert_eq!(symspell.edit_distance("test", "tast"), 1);
+        assert_eq!(symspell.edit_distance("test", "toast"), 2);
+    }
+
+    #[test]
+    fn test_suggestions() {
+        let mut symspell = SymSpell::new(2);
+        symspell.add_word("hello", 1000);
+        symspell.add_word("hell", 500);
+        symspell.add_word("help", 750);
+
+        let suggestions = symspell.suggest("helo", 3);
+        assert!(!suggestions.is_empty());
+        assert_eq!(suggestions[0].term, "hello");
+        assert_eq!(suggestions[0].distance, 1);
+    }
+}