spellkit 0.1.0.pre.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b0bb02947ee896cb9fab3cef53771c6a80e4e14123e518ffacb360b2c136b4d
-  data.tar.gz: 2a5af3e67e414f37fb6ff0a3c3a56a1d771e1480f29380b6196000f9f27e7071
+  metadata.gz: df48970adccf2bea2fae1a69f0e3704c0276bac9ab908a1bc5af6835edde0008
+  data.tar.gz: 28a8f37086361c742f902fa33bfd39003a1fff758b7ee306de1069344cf114d0
 SHA512:
-  metadata.gz: 1cf8d17fbdbcea925c32414e0d746ab4533604941b1603b127667f17f224b5c027f1aeb40b31f712c02a4bc584a1694ca75ef2a717563fe98d5d85d505a427da
-  data.tar.gz: 379eb6fa669ea2f13906cb121a69877414c1338f2b68509f9be55d79c29946eaf293136bbee7a766c75841db2d7fc6527cecbb01550a0ebd4cbfb97bcaf13f7c
+  metadata.gz: fcc1a0678ff2714a8844657f883354594d7bfb5be6e76a2708aa6134fa5ef9ef0a407a4be06a5f727e540ab99e6d21bc086a0bc740d3bd6cc2a5ed7c6d9458b0
+  data.tar.gz: 043b4f1aa59dd2ced9219f11736053d577d8afa7bdc83cb746ab91c4dc4bb1d5fd28c05aee79c09e739a3b1dd89c49ee088b2c189af81148db09ac40e0de1f06

data/README.md

@@ -1,4 +1,4 @@
-# SpellKit
+<img src="/docs/assets/spellkit-wide.png" alt="spellkit" height="160px">
 
 Fast, safe typo correction for search-term extraction, wrapping the SymSpell algorithm in Rust via Magnus.
 
@@ -9,6 +9,26 @@ SpellKit provides:
 - **Sub-millisecond latency** - p95 < 2µs on small dictionaries
 - **Thread-safe** - built with Rust's Arc<RwLock> for safe concurrent access
 
+## Why SpellKit?
+
+### No Runtime Dependencies
+SpellKit is a pure Ruby gem with a Rust extension. Just `gem install spellkit` and you're done. No need to install Aspell, Hunspell, or other system packages. This makes deployment simpler and more reliable across different environments.
+
+### Fast Performance
+Built on the SymSpell algorithm with Rust, SpellKit delivers:
+- **350,000+ operations/second** for spell checking
+- **3.7x faster** than Aspell for correctness checks
+- **40x faster** than Aspell for generating suggestions
+- **p99 latency < 25µs** even under load
+
+See the [Benchmarks](#benchmarks) section for detailed comparisons.
+
+### Production Ready
+- Thread-safe concurrent access
+- Hot reload dictionaries without restarts
+- Instance-based API for multi-domain support
+- Comprehensive error handling
+
 ## Installation
 
 Add to your Gemfile:
@@ -42,13 +62,17 @@ end
 # Or load from local file
 # SpellKit.load!(dictionary: "path/to/dictionary.tsv")
 
+# Check if a word is spelled correctly
+puts SpellKit.correct?("hello")
+# => true
+
 # Get suggestions for a misspelled word
-suggestions = SpellKit.suggest("helo", 5)
+suggestions = SpellKit.suggestions("helllo", 5)
 puts suggestions.inspect
 # => [{"term"=>"hello", "distance"=>1, "freq"=>...}]
 
 # Correct a typo
-corrected = SpellKit.correct_if_unknown("helo")
+corrected = SpellKit.correct("helllo")
 puts corrected
 # => "hello"
 
@@ -76,16 +100,20 @@ SpellKit.load!(dictionary: "https://example.com/dict.tsv")
 # Or from local file
 SpellKit.load!(dictionary: "models/dictionary.tsv", edit_distance: 1)
 
+# Check if a word is correct
+SpellKit.correct?("hello")
+# => true
+
 # Get suggestions
-SpellKit.suggest("lyssis", 5)
+SpellKit.suggestions("lyssis", 5)
 # => [{"term"=>"lysis", "distance"=>1, "freq"=>2000}, ...]
 
 # Correct a typo
-SpellKit.correct_if_unknown("helo")
+SpellKit.correct("helllo")
 # => "hello"
 
 # Batch correction
-tokens = %w[helo wrld ruby]
+tokens = %w[helllo wrld ruby]
 SpellKit.correct_tokens(tokens)
 # => ["hello", "world", "ruby"]
 ```
@@ -119,11 +147,11 @@ SpellKit.load!(
 )
 
 # Use guard: :domain to enable protection
-SpellKit.correct_if_unknown("CDK10", guard: :domain)
+SpellKit.correct("CDK10", guard: :domain)
 # => "CDK10"  # protected, never changed
 
 # Batch correction with guards
-tokens = %w[helo wrld ABC-123 for CDK10]
+tokens = %w[helllo wrld ABC-123 for CDK10]
 SpellKit.correct_tokens(tokens, guard: :domain)
 # => ["hello", "world", "ABC-123", "for", "CDK10"]
 ```
@@ -147,8 +175,8 @@ legal_checker.load!(
 )
 
 # Use them independently
-medical_checker.suggest("lyssis", 5)
-legal_checker.suggest("contractt", 5)
+medical_checker.suggestions("lyssis", 5)
+legal_checker.suggestions("contractt", 5)
 
 # Each maintains its own state
 medical_checker.stats  # Shows medical dictionary stats
@@ -169,7 +197,7 @@ SpellKit.configure do |config|
 end
 
 # This becomes the default instance
-SpellKit.suggest("word", 5)  # Uses configured dictionary
+SpellKit.suggestions("word", 5)  # Uses configured dictionary
 ```
 
 ## Dictionary Format
@@ -238,8 +266,94 @@ SpellKit.load!(
   protected_path: "models/protected.txt",            # optional
   protected_patterns: [/^[A-Z]{3,4}\d+$/],           # optional
   edit_distance: 1,                                  # 1 (default) or 2
-  frequency_threshold: 10.0                          # default: 10.0
+  frequency_threshold: 10.0,                         # default: 10.0 (minimum frequency for corrections)
+
+  # Skip pattern filters (all default to false)
+  skip_urls: true,                                   # Skip URLs (http://, https://, www.)
+  skip_emails: true,                                 # Skip email addresses
+  skip_hostnames: true,                              # Skip hostnames (example.com)
+  skip_code_patterns: true,                          # Skip code identifiers (camelCase, snake_case, etc.)
+  skip_numbers: true                                 # Skip numeric patterns (versions, IDs, measurements)
+)
+```
+
+### Frequency Threshold
+
+The `frequency_threshold` parameter controls which corrections are accepted by `correct` and `correct_tokens`:
+
+- **For misspelled words** (not in dictionary): Only suggest corrections with frequency ≥ `frequency_threshold`
+- **For dictionary words**: Only suggest alternatives with frequency ≥ `frequency_threshold × original_frequency`
+
+This prevents suggesting rare words as corrections for common typos.
+
+**Example:**
+```ruby
+# With default threshold (10.0), suggest any correction with freq ≥ 10
+SpellKit.load!(dictionary: "dict.tsv")
+SpellKit.correct("helllo")  # => "hello" (if freq ≥ 10)
+
+# With high threshold (1000.0), only suggest common corrections
+SpellKit.load!(dictionary: "dict.tsv", frequency_threshold: 1000.0)
+SpellKit.correct("helllo")      # => "hello" (if freq ≥ 1000)
+SpellKit.correct("rarword")   # => "rarword" (no correction if freq < 1000)
+```
+
+### Skip Patterns
+
+SpellKit can automatically skip certain patterns to avoid "correcting" technical terms, URLs, and other special content. Inspired by Aspell's filter modes, these patterns are applied when `guard: :domain` is enabled.
+
+**Available skip patterns:**
+
+```ruby
+SpellKit.load!(
+  dictionary: "dict.tsv",
+  skip_urls: true,           # Skip URLs: https://example.com, www.example.com
+  skip_emails: true,         # Skip emails: user@domain.com, admin+tag@example.com
+  skip_hostnames: true,      # Skip hostnames: example.com, api.example.com
+  skip_code_patterns: true,  # Skip code: camelCase, snake_case, PascalCase, dotted.paths
+  skip_numbers: true         # Skip numbers: 1.2.3, #123, 5kg, 100mb
+)
+
+# With skip patterns enabled, technical content is preserved
+SpellKit.correct("https://example.com", guard: :domain)  # => "https://example.com"
+SpellKit.correct("user@test.com", guard: :domain)        # => "user@test.com"
+SpellKit.correct("getElementById", guard: :domain)       # => "getElementById"
+SpellKit.correct("version-1.2.3", guard: :domain)        # => "version-1.2.3"
+
+# Regular typos are still corrected
+SpellKit.correct("helllo", guard: :domain)               # => "hello"
+```
+
+**What each skip pattern matches:**
+
+- **`skip_urls`**: `http://`, `https://`, `www.` URLs
+- **`skip_emails`**: Email addresses with standard formats including `+` and `.` in usernames
+- **`skip_hostnames`**: Domain names like `example.com`, `api.example.co.uk`
+- **`skip_code_patterns`**:
+  - `camelCase` (starts lowercase)
+  - `PascalCase` (starts uppercase, mixed case)
+  - `snake_case` and `SCREAMING_SNAKE_CASE`
+  - `dotted.paths` like `Array.map` or `config.yml`
+- **`skip_numbers`**:
+  - Version numbers: `1.0`, `2.5.3`, `10.15.7.1`
+  - Hash/IDs: `#123`, `#4567`
+  - Measurements: `5kg`, `2.5m`, `100mb`, `16px`
+  - Words starting with digits: `5test`, `123abc`
+
+**Combining with protected_patterns:**
+
+Skip patterns work alongside your custom `protected_patterns`:
+
+```ruby
+SpellKit.load!(
+  dictionary: "dict.tsv",
+  skip_urls: true,                      # Built-in URL skipping
+  protected_patterns: [/^CUSTOM-\d+$/]  # Your custom patterns
 )
+
+# Both work together
+SpellKit.correct("https://example.com", guard: :domain)  # => "https://example.com" (skip_urls)
+SpellKit.correct("CUSTOM-123", guard: :domain)           # => "CUSTOM-123" (custom pattern)
 ```
 
 ## API Reference
@@ -254,12 +368,24 @@ Load or reload dictionaries. Thread-safe atomic swap. Accepts URLs (auto-downloa
 - `protected_patterns:` (optional) - Array of Regexp or String patterns to protect
 - `edit_distance:` (default: 1) - Maximum edit distance (1 or 2)
 - `frequency_threshold:` (default: 10.0) - Minimum frequency ratio for corrections
+- `skip_urls:` (default: false) - Skip URLs (http://, https://, www.)
+- `skip_emails:` (default: false) - Skip email addresses
+- `skip_hostnames:` (default: false) - Skip hostnames (example.com)
+- `skip_code_patterns:` (default: false) - Skip code identifiers (camelCase, snake_case, etc.)
+- `skip_numbers:` (default: false) - Skip numeric patterns (versions, IDs, measurements)
 
 **Examples:**
 ```ruby
 # From URL (recommended for getting started)
 SpellKit.load!(dictionary: SpellKit::DEFAULT_DICTIONARY_URL)
 
+# With skip patterns for technical content
+SpellKit.load!(
+  dictionary: SpellKit::DEFAULT_DICTIONARY_URL,
+  skip_urls: true,
+  skip_code_patterns: true
+)
+
 # From custom URL
 SpellKit.load!(dictionary: "https://example.com/dict.tsv")
 
@@ -267,7 +393,24 @@ SpellKit.load!(dictionary: "https://example.com/dict.tsv")
 SpellKit.load!(dictionary: "/path/to/dictionary.tsv")
 ```
 
-### `SpellKit.suggest(word, max = 5)`
+### `SpellKit.correct?(word)`
+
+Check if a word is spelled correctly (exact dictionary match).
+
+**Parameters:**
+- `word` (required) - The word to check
+
+**Returns:** Boolean - true if word exists in dictionary, false otherwise
+
+**Performance:** Very fast O(1) HashMap lookup. Use this instead of `suggest()` when you only need to check correctness.
+
+**Example:**
+```ruby
+SpellKit.correct?("hello")  # => true
+SpellKit.correct?("helllo")   # => false
+```
+
+### `SpellKit.suggestions(word, max = 5)`
 
 Get ranked suggestions for a word.
 
@@ -277,16 +420,39 @@ Get ranked suggestions for a word.
 
 **Returns:** Array of hashes with `"term"`, `"distance"`, and `"freq"` keys
 
-### `SpellKit.correct_if_unknown(word, guard:)`
+**Example:**
+```ruby
+SpellKit.suggestions("helllo", 5)
+# => [{"term"=>"hello", "distance"=>1, "freq"=>10000}, ...]
+```
 
-Return corrected word or original if no better match found.
+### `SpellKit.correct(word, guard:)`
 
-**Options:**
-- `guard:` - Set to `:domain` to enable protection checks
+Return corrected word or original if no better match found. Respects `frequency_threshold` configuration.
+
+**Parameters:**
+- `word` (required) - The word to correct
+- `guard:` (optional) - Set to `:domain` to enable protection checks
+
+**Behavior:**
+- Returns original word if it exists in dictionary
+- For misspellings, only accepts corrections with frequency ≥ `frequency_threshold`
+- Returns original word if no corrections pass the threshold
+- When `guard: :domain` is set, protected terms and skip patterns are applied
+
+**Example:**
+```ruby
+SpellKit.correct("helllo")                  # => "hello"
+SpellKit.correct("hello")                   # => "hello" (already correct)
+SpellKit.correct("CDK10", guard: :domain)   # => "CDK10" (protected)
+```
 
 ### `SpellKit.correct_tokens(tokens, guard:)`
 
-Batch correction of an array of tokens.
+Batch correction of an array of tokens. Respects `frequency_threshold` configuration.
+
+**Options:**
+- `guard:` - Set to `:domain` to enable protection checks
 
 **Returns:** Array of corrected strings
 
@@ -377,17 +543,78 @@ end
 
 ## Performance
 
-Benchmarked on M1 MacBook Pro with 20-term test dictionary:
+### SpellKit Standalone (M1 MacBook Pro, Ruby 3.3.0, 80k dictionary)
+
+**Single Word Suggestions:**
+- 3,345 i/s (298.96 μs/i) with max: 1 suggestion
+- 3,198 i/s (312.73 μs/i) with max: 5 suggestions
+- 3,073 i/s (325.45 μs/i) with max: 10 suggestions
+
+**Correction Performance:**
+- `correct`: 1,858 i/s (538.17 μs/i)
+- `correct_tokens` (batch): 2,005 i/s (498.76 μs/i)
+
+**Guard Performance:**
+- Without guard: 2,926 i/s (341.79 μs/i)
+- With guard: 9,337 i/s (107.10 μs/i) - **3.19x faster!**
+  *(Guards short-circuit expensive lookups)*
+
+**Latency Distribution (10,000 iterations):**
+- p50: 61μs
+- p95: 66μs
+- p99: 105μs
+- max: 298μs
+
+**Raw Throughput:** 16,192 ops/sec
+
+### Key Takeaways
+1. **Consistent Performance**: p95 latency of 66μs with 80k dictionary, p99 at 105μs
+2. **Guards are Fast**: Protected term checks improve performance by 3.2x by avoiding dictionary lookups
+3. **High Throughput**: Over 16k operations per second with 80k word dictionary
+4. **Scales Well**: Minimal performance difference between 1 vs 10 suggestions
+
+## Benchmarks
+
+SpellKit includes comprehensive benchmarks to measure performance and compare with other spell checkers.
+
+### Running Benchmarks
+
+**Performance Benchmark** - Comprehensive SpellKit performance analysis:
+```bash
+bundle exec ruby benchmark/performance.rb
+```
+
+Measures:
+- Single word suggestions with varying result limits
+- Correction performance on mixed datasets
+- Batch correction throughput
+- Guard/protection overhead
+- Latency distribution (p50, p95, p99)
+- Raw throughput (ops/sec)
+
+**Aspell Comparison** - Direct comparison with Aspell:
+```bash
+# First install Aspell if needed:
+# macOS: brew install aspell
+# Ubuntu: sudo apt-get install aspell libaspell-dev
+
+bundle exec ruby benchmark/comparison_aspell.rb
+```
+
+Compares SpellKit with Aspell on:
+- Single word correction performance
+- Spell checking (correctness tests)
+- Latency distribution at scale
+
+See [benchmark/README.md](benchmark/README.md) for detailed results and analysis.
+
+### Why These Benchmarks?
 
-- **Load time**: < 100ms
-- **Suggestion latency**: p50 < 2µs, p95 < 2µs
-- **Guard checks**: p95 < 1µs
-- **Memory**: ~150MB for 1M term dictionary (estimated)
+**SpellKit vs Aspell**: Both provide fuzzy matching and suggestions for misspelled words, but use different algorithms:
+- **SpellKit (SymSpell)**: O(1) lookup complexity, optimized for speed with large dictionaries
+- **Aspell**: Statistical scoring with phonetic similarity, good for natural language
 
-Target for production (1-5M terms):
-- Load: < 500ms
-- p50: < 30µs, p95: < 100µs
-- Memory: 50-150MB
+The comparison shows SpellKit's performance advantage while solving the same problem.
 
 ## Building Dictionaries
 

data/ext/spellkit/Cargo.lock

@@ -113,12 +113,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itoa"
-version = "1.0.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
-
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -275,61 +269,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
-[[package]]
-name = "ryu"
-version = "1.0.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
-
 [[package]]
 name = "seq-macro"
 version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
 
-[[package]]
-name = "serde"
-version = "1.0.227"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80ece43fc6fbed4eb5392ab50c07334d3e577cbf40997ee896fe7af40bba4245"
-dependencies = [
- "serde_core",
- "serde_derive",
-]
-
-[[package]]
-name = "serde_core"
-version = "1.0.227"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a576275b607a2c86ea29e410193df32bc680303c82f31e275bbfcafe8b33be5"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.227"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51e694923b8824cf0e9b382adf0f60d4e05f348f357b38833a3fa5ed7c2ede04"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.145"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
-dependencies = [
- "itoa",
- "memchr",
- "ryu",
- "serde",
- "serde_core",
-]
-
 [[package]]
 name = "shell-words"
 version = "1.1.0"
@@ -349,8 +294,6 @@ dependencies = [
  "hashbrown",
  "magnus",
  "regex",
- "serde",
- "serde_json",
  "unicode-normalization",
 ]
 

data/ext/spellkit/Cargo.toml

@@ -12,8 +12,6 @@ crate-type = ["cdylib"]
 
 [dependencies]
 magnus = { version = "0.7", features = ["rb-sys"] }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
 hashbrown = "0.15"
 unicode-normalization = "0.1"
 regex = "1.11"

data/ext/spellkit/src/guards.rs

@@ -1,5 +1,6 @@
 use hashbrown::HashSet;
-use regex::Regex;
+use regex::{Regex, RegexBuilder};
+use crate::symspell::SymSpell;
 
 #[derive(Debug, Clone)]
 pub struct Guards {
@@ -19,14 +20,31 @@ impl Guards {
         for line in content.lines() {
             let trimmed = line.trim();
             if !trimmed.is_empty() && !trimmed.starts_with('#') {
+                // Store literal form
                 self.protected_set.insert(trimmed.to_string());
+                // Store lowercase form
                 self.protected_set.insert(trimmed.to_lowercase());
+                // Store normalized form (strips whitespace, converts to lowercase)
+                // This ensures variants like "newyork" are protected if "New York" is in the list
+                let normalized = SymSpell::normalize_word(trimmed);
+                self.protected_set.insert(normalized);
             }
         }
     }
 
-    pub fn add_pattern(&mut self, pattern: &str) -> Result<(), String> {
-        match Regex::new(pattern) {
+    pub fn add_pattern_with_flags(
+        &mut self,
+        pattern: &str,
+        case_insensitive: bool,
+        multiline: bool,
+        extended: bool,
+    ) -> Result<(), String> {
+        match RegexBuilder::new(pattern)
+            .case_insensitive(case_insensitive)
+            .multi_line(multiline)
+            .ignore_whitespace(extended)
+            .build()
+        {
             Ok(regex) => {
                 self.protected_patterns.push(regex);
                 Ok(())