tokenkit 0.1.0.pre.1

data/ext/tokenkit/src/tokenizer/mod.rs

@@ -0,0 +1,254 @@
+mod base;
+mod whitespace;
+mod unicode;
+mod pattern;
+mod sentence;
+mod grapheme;
+mod keyword;
+mod edge_ngram;
+mod ngram;
+mod path_hierarchy;
+mod url_email;
+mod char_group;
+mod letter;
+mod lowercase;
+
+pub(crate) use base::BaseTokenizerFields;
+
+pub use whitespace::WhitespaceTokenizer;
+pub use unicode::UnicodeTokenizer;
+pub use pattern::PatternTokenizer;
+pub use sentence::SentenceTokenizer;
+pub use grapheme::GraphemeTokenizer;
+pub use keyword::KeywordTokenizer;
+pub use edge_ngram::EdgeNgramTokenizer;
+pub use ngram::NgramTokenizer;
+pub use path_hierarchy::PathHierarchyTokenizer;
+pub use url_email::UrlEmailTokenizer;
+pub use char_group::CharGroupTokenizer;
+pub use letter::LetterTokenizer;
+pub use lowercase::LowercaseTokenizer;
+
+use crate::config::{TokenizerConfig, TokenizerStrategy};
+use crate::error::Result;
+use regex::Regex;
+
+pub trait Tokenizer: Send + Sync {
+    fn tokenize(&self, text: &str) -> Vec<String>;
+}
+
+pub fn from_config(config: TokenizerConfig) -> Result<Box<dyn Tokenizer>> {
+    match config.strategy.clone() {
+        TokenizerStrategy::Whitespace => Ok(Box::new(WhitespaceTokenizer::new(config))),
+        TokenizerStrategy::Unicode => Ok(Box::new(UnicodeTokenizer::new(config))),
+        TokenizerStrategy::Pattern { regex } => {
+            PatternTokenizer::new(&regex, config)
+                .map(|t| Box::new(t) as Box<dyn Tokenizer>)
+        }
+        TokenizerStrategy::Sentence => Ok(Box::new(SentenceTokenizer::new(config))),
+        TokenizerStrategy::Grapheme { extended } => {
+            Ok(Box::new(GraphemeTokenizer::new(config, extended)))
+        }
+        TokenizerStrategy::Keyword => Ok(Box::new(KeywordTokenizer::new(config))),
+        TokenizerStrategy::EdgeNgram { min_gram, max_gram } => {
+            Ok(Box::new(EdgeNgramTokenizer::new(config, min_gram, max_gram)))
+        }
+        TokenizerStrategy::PathHierarchy { delimiter } => {
+            Ok(Box::new(PathHierarchyTokenizer::new(config, delimiter)))
+        }
+        TokenizerStrategy::UrlEmail => {
+            Ok(Box::new(UrlEmailTokenizer::new(config)))
+        }
+        TokenizerStrategy::Ngram { min_gram, max_gram } => {
+            Ok(Box::new(NgramTokenizer::new(config, min_gram, max_gram)))
+        }
+        TokenizerStrategy::CharGroup { split_on_chars } => {
+            Ok(Box::new(CharGroupTokenizer::new(config, split_on_chars)))
+        }
+        TokenizerStrategy::Letter => Ok(Box::new(LetterTokenizer::new(config))),
+        TokenizerStrategy::Lowercase => Ok(Box::new(LowercaseTokenizer::new(config))),
+    }
+}
+
+pub(crate) fn merge_overlapping_spans(mut spans: Vec<(usize, usize, String)>) -> Vec<(usize, usize, String)> {
+    if spans.is_empty() {
+        return spans;
+    }
+
+    spans.sort_by(|a, b| {
+        a.0.cmp(&b.0)
+            .then_with(|| b.1.cmp(&a.1))
+    });
+
+    let mut merged = Vec::new();
+    let mut current = spans[0].clone();
+
+    for span in spans.into_iter().skip(1) {
+        if span.0 < current.1 {
+            if span.1 > current.1 {
+                current = span;
+            }
+        } else {
+            merged.push(current);
+            current = span;
+        }
+    }
+    merged.push(current);
+
+    merged
+}
+
+// Optimized version that works with indices only
+fn merge_overlapping_spans_optimized(mut spans: Vec<(usize, usize)>) -> Vec<(usize, usize)> {
+    if spans.is_empty() {
+        return spans;
+    }
+
+    spans.sort_unstable_by(|a, b| {
+        a.0.cmp(&b.0)
+            .then_with(|| b.1.cmp(&a.1))
+    });
+
+    let mut merged = Vec::with_capacity(spans.len());
+    let mut current = spans[0];
+
+    for span in spans.into_iter().skip(1) {
+        if span.0 < current.1 {
+            if span.1 > current.1 {
+                current.1 = span.1;
+            }
+        } else {
+            merged.push(current);
+            current = span;
+        }
+    }
+    merged.push(current);
+    merged
+}
+
+pub(crate) fn apply_preserve_patterns(
+    tokens: Vec<String>,
+    preserve_patterns: &[Regex],
+    original_text: &str,
+    config: &TokenizerConfig,
+) -> Vec<String> {
+    apply_preserve_patterns_with_tokenizer(
+        tokens,
+        preserve_patterns,
+        original_text,
+        config,
+        tokenize_simple,
+    )
+}
+
+pub(crate) fn apply_preserve_patterns_with_tokenizer<F>(
+    tokens: Vec<String>,
+    preserve_patterns: &[Regex],
+    original_text: &str,
+    config: &TokenizerConfig,
+    tokenizer_fn: F,
+) -> Vec<String>
+where
+    F: Fn(&str) -> Vec<String>,
+{
+    if preserve_patterns.is_empty() {
+        return tokens;
+    }
+
+    // Use indices instead of allocating strings upfront
+    let mut preserved_spans: Vec<(usize, usize)> = Vec::with_capacity(32);
+    for pattern in preserve_patterns {
+        for mat in pattern.find_iter(original_text) {
+            preserved_spans.push((mat.start(), mat.end()));
+        }
+    }
+
+    if preserved_spans.is_empty() {
+        return tokens;
+    }
+
+    let preserved_spans = merge_overlapping_spans_optimized(preserved_spans);
+
+    // Pre-allocate result vector with estimated capacity
+    let mut result = Vec::with_capacity(tokens.len() + preserved_spans.len());
+    let mut pos = 0;
+
+    for (start, end) in preserved_spans {
+        if start > pos {
+            let before = &original_text[pos..start];
+            let mut before_tokens = tokenizer_fn(before);
+            post_process_in_place(&mut before_tokens, config);
+            result.extend(before_tokens);
+        }
+        // Extract preserved text only when needed
+        result.push(original_text[start..end].to_string());
+        pos = end;
+    }
+
+    if pos < original_text.len() {
+        let remaining = &original_text[pos..];
+        let mut remaining_tokens = tokenizer_fn(remaining);
+        post_process_in_place(&mut remaining_tokens, config);
+        result.extend(remaining_tokens);
+    }
+
+    result
+}
+
+fn tokenize_simple(text: &str) -> Vec<String> {
+    text.split_whitespace()
+        .filter(|s| !s.is_empty())
+        .map(|s| s.to_string())
+        .collect()
+}
+
+pub(crate) fn post_process(tokens: Vec<String>, config: &TokenizerConfig) -> Vec<String> {
+    post_process_with_preserved(tokens, config, None)
+}
+
+// In-place version to avoid allocation
+fn post_process_in_place(tokens: &mut Vec<String>, config: &TokenizerConfig) {
+    if config.lowercase {
+        for token in tokens.iter_mut() {
+            *token = token.to_lowercase();
+        }
+    }
+
+    if config.remove_punctuation {
+        tokens.retain_mut(|token| {
+            token.retain(|c| !c.is_ascii_punctuation());
+            !token.is_empty()
+        });
+    }
+}
+
+pub(crate) fn post_process_with_preserved(
+    mut tokens: Vec<String>,
+    config: &TokenizerConfig,
+    preserve_chars: Option<&str>,
+) -> Vec<String> {
+    if config.lowercase {
+        tokens = tokens.into_iter().map(|t| t.to_lowercase()).collect();
+    }
+
+    if config.remove_punctuation {
+        tokens = tokens
+            .into_iter()
+            .map(|t| {
+                t.chars()
+                    .filter(|c| {
+                        if let Some(preserved) = preserve_chars {
+                            if preserved.contains(*c) {
+                                return true;
+                            }
+                        }
+                        !c.is_ascii_punctuation()
+                    })
+                    .collect()
+            })
+            .filter(|s: &String| !s.is_empty())
+            .collect();
+    }
+
+    tokens
+}

data/ext/tokenkit/src/tokenizer/ngram.rs

@@ -0,0 +1,80 @@
+use super::Tokenizer;
+use crate::config::TokenizerConfig;
+
+pub struct NgramTokenizer {
+    config: TokenizerConfig,
+    min_gram: usize,
+    max_gram: usize,
+}
+
+impl NgramTokenizer {
+    pub fn new(config: TokenizerConfig, min_gram: usize, max_gram: usize) -> Self {
+        // Validate and sanitize parameters
+        let min_gram = min_gram.max(1); // Minimum 1 character
+        let max_gram = max_gram.max(min_gram); // Ensure max >= min
+
+        Self {
+            config,
+            min_gram,
+            max_gram,
+        }
+    }
+
+    fn generate_ngrams(&self, text: &str) -> Vec<String> {
+        let mut ngrams = Vec::new();
+        let chars: Vec<char> = text.chars().collect();
+        let text_len = chars.len();
+
+        if text_len == 0 {
+            return ngrams;
+        }
+
+        let max = self.max_gram.min(text_len);
+
+        for gram_size in self.min_gram..=max {
+            for start in 0..=(text_len - gram_size) {
+                let ngram: String = chars.iter().skip(start).take(gram_size).collect();
+                ngrams.push(ngram);
+            }
+        }
+
+        ngrams
+    }
+}
+
+impl Tokenizer for NgramTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let mut all_ngrams = Vec::new();
+
+        for word in text.split_whitespace() {
+            if word.is_empty() {
+                continue;
+            }
+
+            let processed_word = if self.config.remove_punctuation {
+                word.chars()
+                    .filter(|c| !c.is_ascii_punctuation())
+                    .collect()
+            } else {
+                word.to_string()
+            };
+
+            if processed_word.is_empty() {
+                continue;
+            }
+
+            let ngrams = self.generate_ngrams(&processed_word);
+            all_ngrams.extend(ngrams);
+        }
+
+        // Apply lowercase if needed. Note: remove_punctuation already handled above.
+        let mut result = all_ngrams;
+
+        if self.config.lowercase {
+            result = result.into_iter().map(|t| t.to_lowercase()).collect();
+        }
+
+        result
+    }
+
+}

data/ext/tokenkit/src/tokenizer/path_hierarchy.rs

@@ -0,0 +1,187 @@
+use super::{post_process_with_preserved, BaseTokenizerFields, Tokenizer};
+use crate::config::TokenizerConfig;
+
+pub struct PathHierarchyTokenizer {
+    base: BaseTokenizerFields,
+    delimiter: String,
+}
+
+impl PathHierarchyTokenizer {
+    pub fn new(config: TokenizerConfig, delimiter: String) -> Self {
+        Self {
+            base: BaseTokenizerFields::new(config),
+            delimiter,
+        }
+    }
+
+    fn generate_hierarchy(&self, path: &str) -> Vec<String> {
+        let mut tokens = Vec::new();
+        let parts: Vec<&str> = path.split(&self.delimiter).collect();
+
+        let mut current_path = String::new();
+        let starts_with_delimiter = path.starts_with(&self.delimiter);
+
+        for part in parts.iter() {
+            if part.is_empty() {
+                continue;
+            }
+
+            if !current_path.is_empty() {
+                current_path.push_str(&self.delimiter);
+            } else if starts_with_delimiter {
+                current_path.push_str(&self.delimiter);
+            }
+
+            current_path.push_str(part);
+            tokens.push(current_path.clone());
+        }
+
+        tokens
+    }
+
+    fn apply_patterns_to_hierarchy(&self, text: &str) -> Vec<String> {
+        if self.base.preserve_patterns().is_empty() {
+            return self.generate_hierarchy(text);
+        }
+
+        // Generate all hierarchical tokens first
+        let all_tokens = self.generate_hierarchy(text);
+
+        // Find which tokens are completely matched by preserve patterns
+        let mut preserved_tokens = Vec::new();
+        for token in &all_tokens {
+            for pattern in self.base.preserve_patterns() {
+                if let Some(mat) = pattern.find(token) {
+                    if mat.as_str() == token {
+                        preserved_tokens.push(token.clone());
+                        break;
+                    }
+                }
+            }
+        }
+
+        // Now build the result, applying lowercase where appropriate
+        let mut result = Vec::new();
+        for token in all_tokens {
+            // Check if this token should be included
+            // Include if: it's a preserved token OR it extends beyond a preserved token
+            let should_include;
+            let mut apply_lowercase = self.base.config.lowercase;
+
+            if preserved_tokens.contains(&token) {
+                should_include = true;
+                apply_lowercase = false; // Don't lowercase preserved tokens
+            } else {
+                // Check if this token extends a preserved token
+                let mut extends_preserved = false;
+                for preserved in &preserved_tokens {
+                    if token.starts_with(preserved) && token.len() > preserved.len() {
+                        extends_preserved = true;
+                        break;
+                    }
+                }
+
+                if extends_preserved {
+                    should_include = true;
+                } else {
+                    // Include if no preserved token is a prefix of this one
+                    let mut has_preserved_prefix = false;
+                    for preserved in &preserved_tokens {
+                        if preserved.starts_with(&token) && preserved != &token {
+                            has_preserved_prefix = true;
+                            break;
+                        }
+                    }
+                    should_include = !has_preserved_prefix;
+                }
+            }
+
+            if should_include {
+                if apply_lowercase && !preserved_tokens.contains(&token) {
+                    // Apply lowercase to non-preserved parts
+                    let mut lowercased = String::new();
+                    let starts_with_delim = token.starts_with(&self.delimiter);
+                    let parts: Vec<&str> = token.split(&self.delimiter).collect();
+
+                    for (i, part) in parts.iter().enumerate() {
+                        if part.is_empty() {
+                            if i == 0 && starts_with_delim {
+                                // Path starts with delimiter, add it once
+                                lowercased.push_str(&self.delimiter);
+                            }
+                            continue;
+                        }
+
+                        if i > 0 || (i == 0 && starts_with_delim) {
+                            if !lowercased.is_empty() && !lowercased.ends_with(&self.delimiter) {
+                                lowercased.push_str(&self.delimiter);
+                            }
+                        }
+
+                        // Check if this specific part should be preserved
+                        let mut preserve_part = false;
+                        for pattern in self.base.preserve_patterns() {
+                            if pattern.is_match(part) {
+                                preserve_part = true;
+                                break;
+                            }
+                        }
+
+                        if preserve_part {
+                            lowercased.push_str(part);
+                        } else {
+                            lowercased.push_str(&part.to_lowercase());
+                        }
+                    }
+                    result.push(lowercased);
+                } else {
+                    result.push(token);
+                }
+            }
+        }
+
+        result
+    }
+}
+
+impl Tokenizer for PathHierarchyTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let trimmed = text.trim();
+        if trimmed.is_empty() {
+            return vec![];
+        }
+
+        if self.base.has_preserve_patterns() {
+            let mut tokens = self.apply_patterns_to_hierarchy(trimmed);
+
+            // Apply remove_punctuation if needed (but preserve delimiters)
+            if self.base.config.remove_punctuation {
+                tokens = tokens.into_iter().map(|token| {
+                    let parts: Vec<&str> = token.split(&self.delimiter).collect();
+                    let processed: Vec<String> = parts.iter().map(|part| {
+                        if part.is_empty() {
+                            String::new()
+                        } else {
+                            // Check if this part should be preserved
+                            let should_preserve = self.base.preserve_patterns().iter().any(|p| p.is_match(part));
+                            if should_preserve {
+                                part.to_string()
+                            } else {
+                                part.chars()
+                                    .filter(|c| !c.is_ascii_punctuation() || self.delimiter.contains(*c))
+                                    .collect()
+                            }
+                        }
+                    }).collect();
+                    processed.join(&self.delimiter)
+                }).filter(|s| !s.is_empty() && s != &self.delimiter).collect();
+            }
+
+            tokens
+        } else {
+            let tokens = self.generate_hierarchy(trimmed);
+            post_process_with_preserved(tokens, &self.base.config, Some(&self.delimiter))
+        }
+    }
+
+}

data/ext/tokenkit/src/tokenizer/pattern.rs

@@ -0,0 +1,38 @@
+use super::{apply_preserve_patterns, post_process, BaseTokenizerFields, Tokenizer};
+use crate::config::TokenizerConfig;
+use crate::error::Result;
+use regex::Regex;
+
+pub struct PatternTokenizer {
+    base: BaseTokenizerFields,
+    pattern: Regex,
+}
+
+impl PatternTokenizer {
+    pub fn new(regex: &str, config: TokenizerConfig) -> Result<Self> {
+        // Pattern is already validated in validate_config(), safe to unwrap
+        let pattern = Regex::new(regex).expect("Pattern should have been validated");
+
+        Ok(Self {
+            base: BaseTokenizerFields::new(config),
+            pattern,
+        })
+    }
+}
+
+impl Tokenizer for PatternTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let tokens: Vec<String> = self
+            .pattern
+            .find_iter(text)
+            .map(|mat| mat.as_str().to_string())
+            .collect();
+
+        if self.base.has_preserve_patterns() {
+            apply_preserve_patterns(tokens, self.base.preserve_patterns(), text, &self.base.config)
+        } else {
+            post_process(tokens, &self.base.config)
+        }
+    }
+
+}

data/ext/tokenkit/src/tokenizer/sentence.rs

@@ -0,0 +1,89 @@
+use super::{post_process, BaseTokenizerFields, Tokenizer};
+use crate::config::TokenizerConfig;
+use unicode_segmentation::UnicodeSegmentation;
+
+pub struct SentenceTokenizer {
+    base: BaseTokenizerFields,
+}
+
+impl SentenceTokenizer {
+    pub fn new(config: TokenizerConfig) -> Self {
+        Self {
+            base: BaseTokenizerFields::new(config),
+        }
+    }
+}
+
+impl SentenceTokenizer {
+    fn apply_patterns_to_sentence(&self, sentence: &str) -> String {
+        if self.base.preserve_patterns().is_empty() || !self.base.config.lowercase {
+            return sentence.to_string();
+        }
+
+        // Find all matches in the sentence
+        let mut preserved_spans: Vec<(usize, usize, String)> = Vec::new();
+        for pattern in self.base.preserve_patterns() {
+            for mat in pattern.find_iter(sentence) {
+                preserved_spans.push((mat.start(), mat.end(), mat.as_str().to_string()));
+            }
+        }
+
+        if preserved_spans.is_empty() {
+            return sentence.to_string();
+        }
+
+        // Sort and merge overlapping spans
+        preserved_spans.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| b.1.cmp(&a.1)));
+
+        let mut result = String::new();
+        let mut pos = 0;
+
+        for (start, end, preserved) in preserved_spans {
+            if start > pos {
+                // Lowercase the text before the preserved pattern
+                result.push_str(&sentence[pos..start].to_lowercase());
+            }
+            // Keep the preserved pattern as-is
+            result.push_str(&preserved);
+            pos = end.max(pos); // Handle overlaps
+        }
+
+        if pos < sentence.len() {
+            // Lowercase the remaining text
+            result.push_str(&sentence[pos..].to_lowercase());
+        }
+
+        result
+    }
+}
+
+impl Tokenizer for SentenceTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let mut sentences: Vec<String> = text
+            .unicode_sentences()
+            .map(|s| s.to_string())
+            .collect();
+
+        // Apply preserve patterns to each sentence
+        if self.base.has_preserve_patterns() && self.base.config.lowercase {
+            sentences = sentences
+                .into_iter()
+                .map(|sentence| self.apply_patterns_to_sentence(&sentence))
+                .collect();
+
+            // Don't call post_process since we already handled lowercasing with patterns
+            // Just handle remove_punctuation if needed
+            if self.base.config.remove_punctuation {
+                sentences = sentences
+                    .into_iter()
+                    .map(|s| s.chars().filter(|c| !c.is_ascii_punctuation()).collect())
+                    .filter(|s: &String| !s.is_empty())
+                    .collect();
+            }
+            sentences
+        } else {
+            post_process(sentences, &self.base.config)
+        }
+    }
+
+}

data/ext/tokenkit/src/tokenizer/unicode.rs

@@ -0,0 +1,36 @@
+use super::{apply_preserve_patterns, post_process, BaseTokenizerFields, Tokenizer};
+use crate::config::TokenizerConfig;
+use unicode_segmentation::UnicodeSegmentation;
+
+pub struct UnicodeTokenizer {
+    base: BaseTokenizerFields,
+}
+
+impl UnicodeTokenizer {
+    pub fn new(config: TokenizerConfig) -> Self {
+        Self {
+            base: BaseTokenizerFields::new(config),
+        }
+    }
+}
+
+impl Tokenizer for UnicodeTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        if self.base.has_preserve_patterns() {
+            let tokens = text
+                .unicode_words()
+                .map(|s| s.to_string())
+                .collect();
+
+            return apply_preserve_patterns(tokens, self.base.preserve_patterns(), text, &self.base.config);
+        }
+
+        let tokens: Vec<String> = text
+            .unicode_words()
+            .map(|s| s.to_string())
+            .collect();
+
+        post_process(tokens, &self.base.config)
+    }
+
+}