spellkit 0.2.0-aarch64-linux

data/ext/spellkit/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "spellkit"
+version = "0.1.0"
+edition = "2021"
+authors = ["Chris Petersen <chris@petersen.io>"]
+license = "MIT"
+description = "Fast, safe typo correction for search-term extraction"
+
+[lib]
+name = "spellkit"
+crate-type = ["cdylib"]
+
+[dependencies]
+magnus = { version = "0.7", features = ["rb-sys"] }
+hashbrown = "0.15"
+unicode-normalization = "0.1"
+regex = "1.11"
+
+[dev-dependencies]

data/ext/spellkit/extconf.rb

@@ -0,0 +1,4 @@
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("spellkit/spellkit")

data/ext/spellkit/src/guards.rs

@@ -0,0 +1,75 @@
+use hashbrown::HashSet;
+use regex::{Regex, RegexBuilder};
+use crate::symspell::SymSpell;
+
+#[derive(Debug, Clone)]
+pub struct Guards {
+    protected_set: HashSet<String>,
+    protected_patterns: Vec<Regex>,
+}
+
+impl Guards {
+    pub fn new() -> Self {
+        Self {
+            protected_set: HashSet::new(),
+            protected_patterns: Vec::new(),
+        }
+    }
+
+    pub fn load_protected(&mut self, content: &str) {
+        for line in content.lines() {
+            let trimmed = line.trim();
+            if !trimmed.is_empty() && !trimmed.starts_with('#') {
+                // Store literal form
+                self.protected_set.insert(trimmed.to_string());
+                // Store lowercase form
+                self.protected_set.insert(trimmed.to_lowercase());
+                // Store normalized form (strips whitespace, converts to lowercase)
+                // This ensures variants like "newyork" are protected if "New York" is in the list
+                let normalized = SymSpell::normalize_word(trimmed);
+                self.protected_set.insert(normalized);
+            }
+        }
+    }
+
+    pub fn add_pattern_with_flags(
+        &mut self,
+        pattern: &str,
+        case_insensitive: bool,
+        multiline: bool,
+        extended: bool,
+    ) -> Result<(), String> {
+        match RegexBuilder::new(pattern)
+            .case_insensitive(case_insensitive)
+            .multi_line(multiline)
+            .ignore_whitespace(extended)
+            .build()
+        {
+            Ok(regex) => {
+                self.protected_patterns.push(regex);
+                Ok(())
+            }
+            Err(e) => Err(format!("Invalid regex pattern: {}", e)),
+        }
+    }
+
+    pub fn is_protected(&self, word: &str) -> bool {
+        let lower = word.to_lowercase();
+
+        if self.protected_set.contains(word) || self.protected_set.contains(&lower) {
+            return true;
+        }
+
+        for pattern in &self.protected_patterns {
+            if pattern.is_match(word) {
+                return true;
+            }
+        }
+
+        false
+    }
+
+    pub fn is_protected_normalized(&self, word: &str, normalized: &str) -> bool {
+        self.is_protected(word) || self.is_protected(normalized)
+    }
+}

data/ext/spellkit/src/lib.rs

@@ -0,0 +1,393 @@
+mod symspell;
+mod guards;
+
+use magnus::{class, define_module, function, method, prelude::*, Error, RArray, RHash, Ruby, Value, TryConvert};
+use std::sync::{Arc, RwLock};
+use symspell::SymSpell;
+use guards::Guards;
+
+use std::time::{SystemTime, UNIX_EPOCH};
+
+#[derive(Clone)]
+#[magnus::wrap(class = "SpellKit::Checker", free_immediately, size)]
+struct Checker {
+    state: Arc<RwLock<CheckerState>>,
+}
+
+struct CheckerState {
+    symspell: Option<SymSpell>,
+    guards: Guards,
+    loaded: bool,
+    frequency_threshold: f64,
+    loaded_at: Option<u64>,
+    dictionary_size: usize,
+    edit_distance: usize,
+    skipped_malformed: usize,
+    skipped_multiword: usize,
+    skipped_invalid_freq: usize,
+    skipped_duplicates: usize,
+}
+
+impl CheckerState {
+    fn new() -> Self {
+        Self {
+            symspell: None,
+            guards: Guards::new(),
+            loaded: false,
+            frequency_threshold: 10.0,
+            loaded_at: None,
+            dictionary_size: 0,
+            edit_distance: 1,
+            skipped_malformed: 0,
+            skipped_multiword: 0,
+            skipped_invalid_freq: 0,
+            skipped_duplicates: 0,
+        }
+    }
+}
+
+// Helper function to correct a single word
+// Returns the corrected word or the original if no correction is appropriate
+fn correct_word(
+    state: &CheckerState,
+    symspell: &SymSpell,
+    word: &str,
+) -> String {
+    // Always check if word is protected
+    let normalized = SymSpell::normalize_word(word);
+    if state.guards.is_protected_normalized(word, &normalized) {
+        return word.to_string();
+    }
+
+    let suggestions = symspell.suggestions(word, 5);
+
+    // If exact match exists, return canonical form from dictionary
+    if !suggestions.is_empty() && suggestions[0].distance == 0 {
+        return suggestions[0].term.clone();
+    }
+
+    // Get original word's frequency (if it exists in dictionary)
+    let original_freq = symspell.get_frequency(word);
+
+    // Find best correction with frequency threshold
+    for suggestion in &suggestions {
+        if suggestion.distance <= state.edit_distance {
+            // Apply frequency threshold
+            let passes_threshold = match original_freq {
+                // Word not in dictionary: require suggestion frequency >= absolute threshold
+                None => suggestion.frequency as f64 >= state.frequency_threshold,
+                // Word in dictionary: require suggestion frequency >= threshold * original frequency
+                Some(orig_freq) => {
+                    suggestion.frequency as f64 >= state.frequency_threshold * orig_freq as f64
+                }
+            };
+
+            if passes_threshold {
+                return suggestion.term.clone();
+            }
+        }
+    }
+
+    // No suggestions passed the threshold
+    word.to_string()
+}
+
+impl Checker {
+    fn new() -> Self {
+        Self {
+            state: Arc::new(RwLock::new(CheckerState::new())),
+        }
+    }
+
+    fn load_full(&self, config: RHash) -> Result<(), Error> {
+        let ruby = Ruby::get().unwrap();
+
+        // Required: dictionary path
+        let dictionary_path: String = TryConvert::try_convert(
+            config.fetch::<_, Value>("dictionary_path")
+                .map_err(|_| Error::new(ruby.exception_arg_error(), "dictionary_path is required"))?
+        )?;
+
+        // Optional: edit distance
+        let edit_dist: usize = config.get("edit_distance")
+            .and_then(|v: Value| TryConvert::try_convert(v).ok())
+            .unwrap_or(1);
+
+        if edit_dist > 2 {
+            return Err(Error::new(ruby.exception_arg_error(), "edit_distance must be 1 or 2"));
+        }
+
+        // Stream dictionary loading: read line-by-line and add directly to SymSpell
+        // This avoids buffering the entire file and intermediate Vec allocation
+        let file = std::fs::File::open(&dictionary_path)
+            .map_err(|e| Error::new(ruby.exception_runtime_error(), format!("Failed to open dictionary file: {}", e)))?;
+
+        let reader = std::io::BufReader::new(file);
+        let mut symspell = SymSpell::new(edit_dist);
+        let mut dictionary_size = 0;
+        let mut skipped_malformed = 0;
+        let mut skipped_multiword = 0;
+        let mut skipped_invalid_freq = 0;
+        let mut skipped_duplicates = 0;
+
+        use std::io::BufRead;
+        for line in reader.lines() {
+            let line = line.map_err(|e| Error::new(ruby.exception_runtime_error(), format!("Failed to read line: {}", e)))?;
+
+            // Try tab-separated first (allows multi-word terms), then space-separated (SymSpell format)
+            let parts: Vec<&str> = if line.contains('\t') {
+                line.split('\t').collect()
+            } else {
+                line.split_whitespace().collect()
+            };
+
+            // Validate we have exactly 2 columns (term and frequency)
+            if parts.len() != 2 {
+                skipped_malformed += 1;
+                continue;
+            }
+
+            let term = parts[0].trim();
+            let freq_str = parts[1].trim();
+
+            // Skip empty terms or frequencies
+            if term.is_empty() || freq_str.is_empty() {
+                skipped_malformed += 1;
+                continue;
+            }
+
+            // Check for multi-word terms (SymSpell algorithm doesn't support phrases)
+            if term.contains(char::is_whitespace) {
+                skipped_multiword += 1;
+                continue;
+            }
+
+            // Parse frequency
+            match freq_str.parse::<u64>() {
+                Ok(freq) => {
+                    let normalized = SymSpell::normalize_word(term);
+                    let was_new = symspell.add_word(&normalized, term, freq);
+                    if was_new {
+                        dictionary_size += 1;
+                    } else {
+                        skipped_duplicates += 1;
+                    }
+                }
+                Err(_) => {
+                    skipped_invalid_freq += 1;
+                }
+            }
+        }
+
+        let mut guards = Guards::new();
+
+        // Load optional protected terms file
+        if let Some(protected_path) = config.get("protected_path") {
+            let path: String = TryConvert::try_convert(protected_path)?;
+            let content = std::fs::read_to_string(&path)
+                .map_err(|e| Error::new(ruby.exception_runtime_error(),
+                    format!("Failed to read protected terms file '{}': {}", path, e)))?;
+            guards.load_protected(&content);
+        }
+
+        // Load optional protected patterns
+        if let Some(patterns_value) = config.get("protected_patterns") {
+            let patterns: RArray = TryConvert::try_convert(patterns_value)?;
+            for pattern_value in patterns.into_iter() {
+                let pattern_hash: RHash = TryConvert::try_convert(pattern_value)?;
+
+                let source: String = TryConvert::try_convert(
+                    pattern_hash.fetch::<_, Value>("source")
+                        .map_err(|_| Error::new(ruby.exception_arg_error(), "pattern hash missing 'source' key"))?
+                )?;
+
+                let case_insensitive: bool = pattern_hash.get("case_insensitive")
+                    .and_then(|v: Value| TryConvert::try_convert(v).ok())
+                    .unwrap_or(false);
+
+                let multiline: bool = pattern_hash.get("multiline")
+                    .and_then(|v: Value| TryConvert::try_convert(v).ok())
+                    .unwrap_or(false);
+
+                let extended: bool = pattern_hash.get("extended")
+                    .and_then(|v: Value| TryConvert::try_convert(v).ok())
+                    .unwrap_or(false);
+
+                guards.add_pattern_with_flags(&source, case_insensitive, multiline, extended)
+                    .map_err(|e| Error::new(ruby.exception_arg_error(), e))?;
+            }
+        }
+
+        // Optional frequency threshold
+        let frequency_threshold: f64 = config.get("frequency_threshold")
+            .and_then(|v: Value| TryConvert::try_convert(v).ok())
+            .unwrap_or(10.0);
+
+        // Validate frequency threshold
+        if !frequency_threshold.is_finite() {
+            return Err(Error::new(ruby.exception_arg_error(), "frequency_threshold must be finite (not NaN or Infinity)"));
+        }
+
+        if frequency_threshold < 0.0 {
+            return Err(Error::new(ruby.exception_arg_error(), format!("frequency_threshold must be non-negative, got: {}", frequency_threshold)));
+        }
+
+        let loaded_at = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .ok()
+            .map(|d| d.as_secs());
+
+        let mut state = self.state.write().unwrap();
+        state.symspell = Some(symspell);
+        state.guards = guards;
+        state.frequency_threshold = frequency_threshold;
+        state.loaded = true;
+        state.loaded_at = loaded_at;
+        state.dictionary_size = dictionary_size;
+        state.edit_distance = edit_dist;
+        state.skipped_malformed = skipped_malformed;
+        state.skipped_multiword = skipped_multiword;
+        state.skipped_invalid_freq = skipped_invalid_freq;
+        state.skipped_duplicates = skipped_duplicates;
+
+        Ok(())
+    }
+
+    fn suggestions(&self, word: String, max: Option<usize>) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let max_suggestions = max.unwrap_or(5);
+        let state = self.state.read().unwrap();
+
+        if !state.loaded {
+            return Err(Error::new(ruby.exception_runtime_error(), "Dictionary not loaded. Call load! first"));
+        }
+
+        if let Some(ref symspell) = state.symspell {
+            let suggestions = symspell.suggestions(&word, max_suggestions);
+            let result = RArray::new();
+
+            for suggestion in suggestions {
+                let hash = RHash::new();
+                hash.aset("term", suggestion.term)?;
+                hash.aset("distance", suggestion.distance)?;
+                hash.aset("freq", suggestion.frequency)?;
+                result.push(hash)?;
+            }
+
+            Ok(result)
+        } else {
+            Err(Error::new(ruby.exception_runtime_error(), "SymSpell not initialized"))
+        }
+    }
+
+    fn correct(&self, word: String) -> Result<bool, Error> {
+        let ruby = Ruby::get().unwrap();
+        let state = self.state.read().unwrap();
+
+        if !state.loaded {
+            return Err(Error::new(ruby.exception_runtime_error(), "Dictionary not loaded. Call load! first"));
+        }
+
+        if let Some(ref symspell) = state.symspell {
+            Ok(symspell.contains(&word))
+        } else {
+            Err(Error::new(ruby.exception_runtime_error(), "SymSpell not initialized"))
+        }
+    }
+
+    fn correct_if_unknown(&self, word: String) -> Result<String, Error> {
+        let ruby = Ruby::get().unwrap();
+        let state = self.state.read().unwrap();
+
+        if !state.loaded {
+            return Err(Error::new(ruby.exception_runtime_error(), "Dictionary not loaded. Call load! first"));
+        }
+
+        if let Some(ref symspell) = state.symspell {
+            Ok(correct_word(&state, symspell, &word))
+        } else {
+            Err(Error::new(ruby.exception_runtime_error(), "SymSpell not initialized"))
+        }
+    }
+
+    fn correct_tokens(&self, tokens: RArray) -> Result<RArray, Error> {
+        // Optimize batch correction by acquiring lock once for all tokens
+        // instead of calling correct_if_unknown per token (which re-locks each time)
+        let ruby = Ruby::get().unwrap();
+        let state = self.state.read().unwrap();
+
+        if !state.loaded {
+            return Err(Error::new(ruby.exception_runtime_error(), "Dictionary not loaded. Call load! first"));
+        }
+
+        let result = RArray::new();
+
+        if let Some(ref symspell) = state.symspell {
+            for token in tokens.into_iter() {
+                let word: String = TryConvert::try_convert(token)?;
+                let corrected = correct_word(&state, symspell, &word);
+                result.push(corrected)?;
+            }
+
+            Ok(result)
+        } else {
+            Err(Error::new(ruby.exception_runtime_error(), "SymSpell not initialized"))
+        }
+    }
+
+    fn stats(&self) -> Result<RHash, Error> {
+        let state = self.state.read().unwrap();
+        let stats = RHash::new();
+
+        if !state.loaded {
+            stats.aset("loaded", false)?;
+            return Ok(stats);
+        }
+
+        stats.aset("loaded", true)?;
+        stats.aset("dictionary_size", state.dictionary_size)?;
+        stats.aset("edit_distance", state.edit_distance)?;
+        stats.aset("skipped_malformed", state.skipped_malformed)?;
+        stats.aset("skipped_multiword", state.skipped_multiword)?;
+        stats.aset("skipped_invalid_freq", state.skipped_invalid_freq)?;
+        stats.aset("skipped_duplicates", state.skipped_duplicates)?;
+
+        if let Some(loaded_at) = state.loaded_at {
+            stats.aset("loaded_at", loaded_at)?;
+        }
+
+        Ok(stats)
+    }
+
+    fn healthcheck(&self) -> Result<(), Error> {
+        let ruby = Ruby::get().unwrap();
+        let state = self.state.read().unwrap();
+
+        if !state.loaded {
+            return Err(Error::new(ruby.exception_runtime_error(), "Dictionary not loaded"));
+        }
+
+        if state.symspell.is_none() {
+            return Err(Error::new(ruby.exception_runtime_error(), "SymSpell not initialized"));
+        }
+
+        Ok(())
+    }
+}
+
+#[magnus::init]
+fn init(_ruby: &Ruby) -> Result<(), Error> {
+    let module = define_module("SpellKit")?;
+    let checker_class = module.define_class("Checker", class::object())?;
+
+    checker_class.define_singleton_method("new", function!(Checker::new, 0))?;
+    checker_class.define_method("load!", method!(Checker::load_full, 1))?;
+    checker_class.define_method("suggestions", method!(Checker::suggestions, 2))?;
+    checker_class.define_method("correct?", method!(Checker::correct, 1))?;
+    checker_class.define_method("correct", method!(Checker::correct_if_unknown, 1))?;
+    checker_class.define_method("correct_tokens", method!(Checker::correct_tokens, 1))?;
+    checker_class.define_method("stats", method!(Checker::stats, 0))?;
+    checker_class.define_method("healthcheck", method!(Checker::healthcheck, 0))?;
+
+    Ok(())
+}