tokenkit 0.1.0.pre.1

data/ext/tokenkit/src/lib.rs

@@ -0,0 +1,346 @@
+mod config;
+mod error;
+mod tokenizer;
+
+use config::{TokenizerConfig, TokenizerStrategy};
+use error::TokenizerError;
+use magnus::{define_module, function, Error, RArray, RHash, TryConvert};
+use std::sync::Mutex;
+use once_cell::sync::Lazy;
+
+// Store the default configuration and a cached tokenizer
+struct TokenizerCache {
+    config: TokenizerConfig,
+    tokenizer: Option<Box<dyn tokenizer::Tokenizer + Send + Sync>>,
+}
+
+static DEFAULT_CACHE: Lazy<Mutex<TokenizerCache>> = Lazy::new(|| {
+    Mutex::new(TokenizerCache {
+        config: TokenizerConfig::default(),
+        tokenizer: None,
+    })
+});
+
+// Use cached tokenizer if config hasn't changed
+fn tokenize(text: String) -> std::result::Result<Vec<String>, Error> {
+    let mut cache = DEFAULT_CACHE
+        .lock()
+        .map_err(|e| TokenizerError::MutexError(e.to_string()))?;
+
+    // Check if we need to create a new tokenizer
+    if cache.tokenizer.is_none() {
+        let tokenizer = tokenizer::from_config(cache.config.clone())?;
+        cache.tokenizer = Some(tokenizer);
+    }
+
+    // Use the cached tokenizer
+    let result = cache
+        .tokenizer
+        .as_ref()
+        .unwrap()
+        .tokenize(&text);
+
+    Ok(result)
+}
+
+// Configure sets the default configuration and invalidates cache
+fn configure(config_hash: RHash) -> std::result::Result<(), Error> {
+    let config = parse_config_from_hash(config_hash)?;
+
+    // Update cache with new config and clear tokenizer
+    let mut cache = DEFAULT_CACHE
+        .lock()
+        .map_err(|e| TokenizerError::MutexError(e.to_string()))?;
+    cache.config = config;
+    cache.tokenizer = None; // Invalidate cached tokenizer
+
+    Ok(())
+}
+
+// Reset to factory defaults
+fn reset() -> std::result::Result<(), Error> {
+    let mut cache = DEFAULT_CACHE
+        .lock()
+        .map_err(|e| TokenizerError::MutexError(e.to_string()))?;
+    cache.config = TokenizerConfig::default();
+    cache.tokenizer = None; // Clear cached tokenizer
+    Ok(())
+}
+
+// Get current default configuration
+fn config_hash() -> std::result::Result<RHash, Error> {
+    let cache = DEFAULT_CACHE
+        .lock()
+        .map_err(|e| TokenizerError::MutexError(e.to_string()))?;
+
+    config_to_hash(&cache.config)
+}
+
+// Helper function to convert config to RHash
+fn config_to_hash(config: &TokenizerConfig) -> std::result::Result<RHash, Error> {
+    let hash = RHash::new();
+
+    let strategy_str = match &config.strategy {
+        TokenizerStrategy::Whitespace => "whitespace",
+        TokenizerStrategy::Unicode => "unicode",
+        TokenizerStrategy::Pattern { .. } => "pattern",
+        TokenizerStrategy::Sentence => "sentence",
+        TokenizerStrategy::Grapheme { .. } => "grapheme",
+        TokenizerStrategy::Keyword => "keyword",
+        TokenizerStrategy::EdgeNgram { .. } => "edge_ngram",
+        TokenizerStrategy::Ngram { .. } => "ngram",
+        TokenizerStrategy::PathHierarchy { .. } => "path_hierarchy",
+        TokenizerStrategy::UrlEmail => "url_email",
+        TokenizerStrategy::CharGroup { .. } => "char_group",
+        TokenizerStrategy::Letter => "letter",
+        TokenizerStrategy::Lowercase => "lowercase",
+    };
+    hash.aset("strategy", strategy_str)?;
+
+    if let TokenizerStrategy::Pattern { regex } = &config.strategy {
+        hash.aset("regex", regex.as_str())?;
+    }
+
+    if let TokenizerStrategy::Grapheme { extended } = &config.strategy {
+        hash.aset("extended", *extended)?;
+    }
+
+    if let TokenizerStrategy::EdgeNgram { min_gram, max_gram } = &config.strategy {
+        hash.aset("min_gram", *min_gram)?;
+        hash.aset("max_gram", *max_gram)?;
+    }
+
+    if let TokenizerStrategy::PathHierarchy { delimiter } = &config.strategy {
+        hash.aset("delimiter", delimiter.as_str())?;
+    }
+
+    if let TokenizerStrategy::Ngram { min_gram, max_gram } = &config.strategy {
+        hash.aset("min_gram", *min_gram)?;
+        hash.aset("max_gram", *max_gram)?;
+    }
+
+    if let TokenizerStrategy::CharGroup { split_on_chars } = &config.strategy {
+        hash.aset("split_on_chars", split_on_chars.as_str())?;
+    }
+
+    hash.aset("lowercase", config.lowercase)?;
+    hash.aset("remove_punctuation", config.remove_punctuation)?;
+
+    let patterns = RArray::new();
+    for pattern in &config.preserve_patterns {
+        patterns.push(pattern.as_str())?;
+    }
+    hash.aset("preserve_patterns", patterns)?;
+
+    Ok(hash)
+}
+
+// Parse config from Ruby hash
+fn parse_config_from_hash(config_hash: RHash) -> std::result::Result<TokenizerConfig, Error> {
+    let strategy_val = config_hash.get("strategy");
+    let strategy = if let Some(val) = strategy_val {
+        let strategy_str: String = TryConvert::try_convert(val)?;
+        match strategy_str.as_str() {
+            "whitespace" => TokenizerStrategy::Whitespace,
+            "unicode" => TokenizerStrategy::Unicode,
+            "pattern" => {
+                let regex_val = config_hash
+                    .get("regex")
+                    .ok_or_else(|| {
+                        TokenizerError::InvalidConfiguration(
+                            "pattern strategy requires regex parameter".to_string()
+                        )
+                    })?;
+                let regex: String = TryConvert::try_convert(regex_val)?;
+                TokenizerStrategy::Pattern { regex }
+            }
+            "sentence" => TokenizerStrategy::Sentence,
+            "grapheme" => {
+                let extended_val = config_hash.get("extended");
+                let extended = if let Some(val) = extended_val {
+                    TryConvert::try_convert(val)?
+                } else {
+                    true
+                };
+                TokenizerStrategy::Grapheme { extended }
+            }
+            "keyword" => TokenizerStrategy::Keyword,
+            "edge_ngram" => {
+                let min_gram_val = config_hash.get("min_gram");
+                let min_gram = if let Some(val) = min_gram_val {
+                    TryConvert::try_convert(val)?
+                } else {
+                    2
+                };
+                let max_gram_val = config_hash.get("max_gram");
+                let max_gram = if let Some(val) = max_gram_val {
+                    TryConvert::try_convert(val)?
+                } else {
+                    10
+                };
+                TokenizerStrategy::EdgeNgram { min_gram, max_gram }
+            }
+            "path_hierarchy" => {
+                let delimiter_val = config_hash.get("delimiter");
+                let delimiter = if let Some(val) = delimiter_val {
+                    TryConvert::try_convert(val)?
+                } else {
+                    "/".to_string()
+                };
+                TokenizerStrategy::PathHierarchy { delimiter }
+            }
+            "url_email" => TokenizerStrategy::UrlEmail,
+            "ngram" => {
+                let min_gram_val = config_hash.get("min_gram");
+                let min_gram = if let Some(val) = min_gram_val {
+                    TryConvert::try_convert(val)?
+                } else {
+                    2
+                };
+                let max_gram_val = config_hash.get("max_gram");
+                let max_gram = if let Some(val) = max_gram_val {
+                    TryConvert::try_convert(val)?
+                } else {
+                    10
+                };
+                TokenizerStrategy::Ngram { min_gram, max_gram }
+            }
+            "char_group" => {
+                let split_on_chars_val = config_hash.get("split_on_chars");
+                let split_on_chars = if let Some(val) = split_on_chars_val {
+                    TryConvert::try_convert(val)?
+                } else {
+                    " \t\n\r".to_string()
+                };
+                TokenizerStrategy::CharGroup { split_on_chars }
+            }
+            "letter" => TokenizerStrategy::Letter,
+            "lowercase" => TokenizerStrategy::Lowercase,
+            _ => {
+                return Err(TokenizerError::UnknownStrategy(strategy_str).into())
+            }
+        }
+    } else {
+        TokenizerStrategy::Unicode
+    };
+
+    let lowercase_val = config_hash.get("lowercase");
+    let lowercase = if let Some(val) = lowercase_val {
+        TryConvert::try_convert(val)?
+    } else {
+        true
+    };
+
+    let remove_punctuation_val = config_hash.get("remove_punctuation");
+    let remove_punctuation = if let Some(val) = remove_punctuation_val {
+        TryConvert::try_convert(val)?
+    } else {
+        false
+    };
+
+    let preserve_patterns_val = config_hash.get("preserve_patterns");
+    let preserve_patterns = if let Some(val) = preserve_patterns_val {
+        let array: RArray = TryConvert::try_convert(val)?;
+        let mut patterns = Vec::new();
+        for idx in 0..array.len() {
+            let item = array.entry(idx as isize)?;
+            let pattern_str: String = TryConvert::try_convert(item)?;
+            patterns.push(pattern_str);
+        }
+        patterns
+    } else {
+        Vec::new()
+    };
+
+    let config = TokenizerConfig {
+        strategy,
+        lowercase,
+        remove_punctuation,
+        preserve_patterns,
+    };
+
+    // Validate the configuration
+    validate_config(&config)?;
+
+    Ok(config)
+}
+
+// Validate configuration parameters
+fn validate_config(config: &TokenizerConfig) -> std::result::Result<(), TokenizerError> {
+    use TokenizerStrategy::*;
+
+    match &config.strategy {
+        EdgeNgram { min_gram, max_gram } | Ngram { min_gram, max_gram } => {
+            if *min_gram == 0 {
+                return Err(TokenizerError::InvalidNgramConfig {
+                    min: *min_gram,
+                    max: *max_gram,
+                });
+            }
+            if min_gram > max_gram {
+                return Err(TokenizerError::InvalidNgramConfig {
+                    min: *min_gram,
+                    max: *max_gram,
+                });
+            }
+        }
+        PathHierarchy { delimiter } => {
+            if delimiter.is_empty() {
+                return Err(TokenizerError::EmptyDelimiter {
+                    tokenizer: "PathHierarchy".to_string(),
+                });
+            }
+        }
+        Pattern { regex } => {
+            // Validate regex pattern
+            regex::Regex::new(regex).map_err(|e| TokenizerError::InvalidRegex {
+                pattern: regex.clone(),
+                error: e.to_string(),
+            })?;
+        }
+        _ => {}
+    }
+
+    // Validate preserve patterns
+    for pattern in &config.preserve_patterns {
+        regex::Regex::new(pattern).map_err(|e| TokenizerError::InvalidRegex {
+            pattern: pattern.clone(),
+            error: e.to_string(),
+        })?;
+    }
+
+    Ok(())
+}
+
+// Load config is just an alias for configure (for backward compat)
+fn load_config(config_hash: RHash) -> std::result::Result<(), Error> {
+    configure(config_hash)
+}
+
+// Tokenize with a specific config (creates fresh tokenizer)
+fn tokenize_with_config(text: String, config_hash: RHash) -> std::result::Result<Vec<String>, Error> {
+    let config = parse_config_from_hash(config_hash)?;
+
+    // Create fresh tokenizer from config
+    let tokenizer = tokenizer::from_config(config)?;
+
+    // Tokenize and return
+    Ok(tokenizer.tokenize(&text))
+}
+
+#[magnus::init]
+fn init(_ruby: &magnus::Ruby) -> std::result::Result<(), Error> {
+    let module = define_module("TokenKit")?;
+
+    // Public API functions
+    module.define_module_function("_tokenize", function!(tokenize, 1))?;
+    module.define_module_function("_configure", function!(configure, 1))?;
+    module.define_module_function("_reset", function!(reset, 0))?;
+    module.define_module_function("_config_hash", function!(config_hash, 0))?;
+    module.define_module_function("_load_config", function!(load_config, 1))?;
+
+    // New instance-based function
+    module.define_module_function("_tokenize_with_config", function!(tokenize_with_config, 2))?;
+
+    Ok(())
+}

data/ext/tokenkit/src/tokenizer/base.rs

@@ -0,0 +1,41 @@
+use crate::config::TokenizerConfig;
+use regex::Regex;
+
+/// Common functionality for tokenizers that support preserve_patterns
+/// Note: Since we validate patterns in validate_config(), they're guaranteed to be valid here
+pub fn create_preserve_patterns(config: &TokenizerConfig) -> Vec<Regex> {
+    config
+        .preserve_patterns
+        .iter()
+        .map(|p| {
+            // Safe to unwrap because patterns are validated in validate_config()
+            Regex::new(p).expect("Pattern should have been validated")
+        })
+        .collect()
+}
+
+/// Base fields that most tokenizers need
+pub struct BaseTokenizerFields {
+    pub config: TokenizerConfig,
+    pub preserve_patterns: Vec<Regex>,
+}
+
+impl BaseTokenizerFields {
+    pub fn new(config: TokenizerConfig) -> Self {
+        let preserve_patterns = create_preserve_patterns(&config);
+        Self {
+            config,
+            preserve_patterns,
+        }
+    }
+
+
+    pub fn has_preserve_patterns(&self) -> bool {
+        !self.preserve_patterns.is_empty()
+    }
+
+    pub fn preserve_patterns(&self) -> &[Regex] {
+        &self.preserve_patterns
+    }
+}
+

data/ext/tokenkit/src/tokenizer/char_group.rs

@@ -0,0 +1,62 @@
+use super::{apply_preserve_patterns_with_tokenizer, post_process, BaseTokenizerFields, Tokenizer};
+use crate::config::TokenizerConfig;
+use std::collections::HashSet;
+
+pub struct CharGroupTokenizer {
+    base: BaseTokenizerFields,
+    split_chars: HashSet<char>,
+}
+
+impl CharGroupTokenizer {
+    pub fn new(config: TokenizerConfig, split_on_chars: String) -> Self {
+        // Note: Empty split_on_chars is valid - it makes the tokenizer behave like
+        // a keyword tokenizer (no splitting, returns whole text as single token)
+        let split_chars: HashSet<char> = split_on_chars.chars().collect();
+
+        Self {
+            base: BaseTokenizerFields::new(config),
+            split_chars,
+        }
+    }
+
+    fn tokenize_text(&self, text: &str) -> Vec<String> {
+        let mut tokens = Vec::new();
+        let mut current_token = String::new();
+
+        for ch in text.chars() {
+            if self.split_chars.contains(&ch) {
+                if !current_token.is_empty() {
+                    tokens.push(current_token.clone());
+                    current_token.clear();
+                }
+            } else {
+                current_token.push(ch);
+            }
+        }
+
+        if !current_token.is_empty() {
+            tokens.push(current_token);
+        }
+
+        tokens
+    }
+}
+
+impl Tokenizer for CharGroupTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let tokens = self.tokenize_text(text);
+
+        if self.base.has_preserve_patterns() {
+            apply_preserve_patterns_with_tokenizer(
+                tokens,
+                self.base.preserve_patterns(),
+                text,
+                &self.base.config,
+                |t| self.tokenize_text(t),
+            )
+        } else {
+            post_process(tokens, &self.base.config)
+        }
+    }
+
+}

data/ext/tokenkit/src/tokenizer/edge_ngram.rs

@@ -0,0 +1,73 @@
+use super::{Tokenizer};
+use crate::config::TokenizerConfig;
+
+pub struct EdgeNgramTokenizer {
+    config: TokenizerConfig,
+    min_gram: usize,
+    max_gram: usize,
+}
+
+impl EdgeNgramTokenizer {
+    pub fn new(config: TokenizerConfig, min_gram: usize, max_gram: usize) -> Self {
+        // Validate and sanitize parameters
+        let min_gram = min_gram.max(1); // Minimum 1 character
+        let max_gram = max_gram.max(min_gram); // Ensure max >= min
+
+        Self { config, min_gram, max_gram }
+    }
+
+    fn generate_edge_ngrams(&self, text: &str) -> Vec<String> {
+        let mut ngrams = Vec::new();
+        let chars: Vec<char> = text.chars().collect();
+        let text_len = chars.len();
+
+        if text_len == 0 {
+            return ngrams;
+        }
+
+        let max = self.max_gram.min(text_len);
+
+        for gram_size in self.min_gram..=max {
+            let ngram: String = chars.iter().take(gram_size).collect();
+            ngrams.push(ngram);
+        }
+
+        ngrams
+    }
+}
+
+impl Tokenizer for EdgeNgramTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let mut all_ngrams = Vec::new();
+
+        for word in text.split_whitespace() {
+            if word.is_empty() {
+                continue;
+            }
+
+            let processed_word = if self.config.remove_punctuation {
+                word.chars()
+                    .filter(|c| !c.is_ascii_punctuation())
+                    .collect()
+            } else {
+                word.to_string()
+            };
+
+            if processed_word.is_empty() {
+                continue;
+            }
+
+            let ngrams = self.generate_edge_ngrams(&processed_word);
+            all_ngrams.extend(ngrams);
+        }
+
+        let mut result = all_ngrams;
+
+        if self.config.lowercase {
+            result = result.into_iter().map(|t| t.to_lowercase()).collect();
+        }
+
+        result
+    }
+
+}

data/ext/tokenkit/src/tokenizer/grapheme.rs

@@ -0,0 +1,26 @@
+use super::{post_process, Tokenizer};
+use crate::config::TokenizerConfig;
+use unicode_segmentation::UnicodeSegmentation;
+
+pub struct GraphemeTokenizer {
+    config: TokenizerConfig,
+    extended: bool,
+}
+
+impl GraphemeTokenizer {
+    pub fn new(config: TokenizerConfig, extended: bool) -> Self {
+        Self { config, extended }
+    }
+}
+
+impl Tokenizer for GraphemeTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let graphemes: Vec<String> = text
+            .graphemes(self.extended)
+            .map(|s| s.to_string())
+            .collect();
+
+        post_process(graphemes, &self.config)
+    }
+
+}

data/ext/tokenkit/src/tokenizer/keyword.rs

@@ -0,0 +1,25 @@
+use super::{post_process, Tokenizer};
+use crate::config::TokenizerConfig;
+
+pub struct KeywordTokenizer {
+    config: TokenizerConfig,
+}
+
+impl KeywordTokenizer {
+    pub fn new(config: TokenizerConfig) -> Self {
+        Self { config }
+    }
+}
+
+impl Tokenizer for KeywordTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let trimmed = text.trim();
+        if trimmed.is_empty() {
+            return vec![];
+        }
+
+        let tokens = vec![trimmed.to_string()];
+        post_process(tokens, &self.config)
+    }
+
+}

data/ext/tokenkit/src/tokenizer/letter.rs

@@ -0,0 +1,41 @@
+use super::{apply_preserve_patterns, post_process, BaseTokenizerFields, Tokenizer};
+use crate::config::TokenizerConfig;
+
+pub struct LetterTokenizer {
+    base: BaseTokenizerFields,
+}
+
+impl LetterTokenizer {
+    pub fn new(config: TokenizerConfig) -> Self {
+        Self {
+            base: BaseTokenizerFields::new(config),
+        }
+    }
+}
+
+impl Tokenizer for LetterTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let mut tokens = Vec::new();
+        let mut current_token = String::new();
+
+        for ch in text.chars() {
+            if ch.is_alphabetic() {
+                current_token.push(ch);
+            } else if !current_token.is_empty() {
+                tokens.push(current_token.clone());
+                current_token.clear();
+            }
+        }
+
+        if !current_token.is_empty() {
+            tokens.push(current_token);
+        }
+
+        if self.base.has_preserve_patterns() {
+            apply_preserve_patterns(tokens, self.base.preserve_patterns(), text, &self.base.config)
+        } else {
+            post_process(tokens, &self.base.config)
+        }
+    }
+
+}

data/ext/tokenkit/src/tokenizer/lowercase.rs

@@ -0,0 +1,51 @@
+use super::{apply_preserve_patterns, BaseTokenizerFields, Tokenizer};
+use crate::config::TokenizerConfig;
+
+pub struct LowercaseTokenizer {
+    base: BaseTokenizerFields,
+}
+
+impl LowercaseTokenizer {
+    pub fn new(config: TokenizerConfig) -> Self {
+        Self {
+            base: BaseTokenizerFields::new(config),
+        }
+    }
+}
+
+impl Tokenizer for LowercaseTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let mut tokens = Vec::new();
+        let mut current_token = String::new();
+
+        for ch in text.chars() {
+            if ch.is_alphabetic() {
+                for lowercase_ch in ch.to_lowercase() {
+                    current_token.push(lowercase_ch);
+                }
+            } else if !current_token.is_empty() {
+                tokens.push(current_token.clone());
+                current_token.clear();
+            }
+        }
+
+        if !current_token.is_empty() {
+            tokens.push(current_token);
+        }
+
+        // Lowercase tokenizer always lowercases, ignore config.lowercase
+        // Note: remove_punctuation has no effect since we already split on non-alphabetic
+        // characters, but we keep it for consistency with the Tokenizer interface
+
+        if self.base.has_preserve_patterns() {
+            // For preserve_patterns, we need to pass a modified config that doesn't lowercase
+            // because apply_preserve_patterns handles lowercasing for non-preserved tokens
+            let mut modified_config = self.base.config.clone();
+            modified_config.lowercase = true; // Force lowercase for non-preserved tokens
+            apply_preserve_patterns(tokens, self.base.preserve_patterns(), text, &modified_config)
+        } else {
+            tokens
+        }
+    }
+
+}