tokenkit 0.1.0.pre.1

data/ext/tokenkit/src/tokenizer/url_email.rs

@@ -0,0 +1,108 @@
+use super::{merge_overlapping_spans, post_process, Tokenizer};
+use crate::config::TokenizerConfig;
+use linkify::{LinkFinder, LinkKind};
+use regex::Regex;
+use unicode_segmentation::UnicodeSegmentation;
+
+pub struct UrlEmailTokenizer {
+    config: TokenizerConfig,
+    preserve_patterns: Vec<Regex>,
+}
+
+impl UrlEmailTokenizer {
+    pub fn new(config: TokenizerConfig) -> Self {
+        let preserve_patterns = config
+            .preserve_patterns
+            .iter()
+            .filter_map(|p| Regex::new(p).ok())
+            .collect();
+
+        Self {
+            config,
+            preserve_patterns,
+        }
+    }
+
+    fn extract_url_email_spans(&self, text: &str) -> Vec<(usize, usize, String)> {
+        let finder = LinkFinder::new();
+        let mut spans = Vec::new();
+
+        for link in finder.links(text) {
+            match link.kind() {
+                LinkKind::Url | LinkKind::Email => {
+                    let (start, end) = (link.start(), link.end());
+                    spans.push((start, end, link.as_str().to_string()));
+                }
+                _ => {}
+            }
+        }
+
+        spans
+    }
+}
+
+impl Tokenizer for UrlEmailTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let mut spans = self.extract_url_email_spans(text);
+
+        // Add preserve_pattern matches to spans
+        for pattern in &self.preserve_patterns {
+            for mat in pattern.find_iter(text) {
+                spans.push((mat.start(), mat.end(), mat.as_str().to_string()));
+            }
+        }
+
+        // Merge overlapping spans to handle conflicts
+        let spans = if !spans.is_empty() {
+            merge_overlapping_spans(spans)
+        } else {
+            spans
+        };
+
+        if spans.is_empty() {
+            let tokens: Vec<String> = text
+                .unicode_words()
+                .map(|s| s.to_string())
+                .collect();
+            return post_process(tokens, &self.config);
+        }
+
+        let mut result = Vec::new();
+        let mut pos = 0;
+
+        for (start, end, url_or_email) in spans {
+            if start > pos {
+                let before = &text[pos..start];
+                let before_tokens: Vec<String> = before
+                    .unicode_words()
+                    .map(|s| s.to_string())
+                    .collect();
+                let before_tokens = post_process(before_tokens, &self.config);
+                result.extend(before_tokens);
+            }
+
+            // Don't lowercase preserved patterns, but do lowercase URLs/emails if config says so
+            // unless they are from preserve_patterns
+            let preserved = if self.config.lowercase && !self.preserve_patterns.iter().any(|p| p.is_match(&url_or_email)) {
+                url_or_email.to_lowercase()
+            } else {
+                url_or_email
+            };
+            result.push(preserved);
+            pos = end;
+        }
+
+        if pos < text.len() {
+            let remaining = &text[pos..];
+            let remaining_tokens: Vec<String> = remaining
+                .unicode_words()
+                .map(|s| s.to_string())
+                .collect();
+            let remaining_tokens = post_process(remaining_tokens, &self.config);
+            result.extend(remaining_tokens);
+        }
+
+        result
+    }
+
+}

data/ext/tokenkit/src/tokenizer/whitespace.rs

@@ -0,0 +1,31 @@
+use super::{apply_preserve_patterns, post_process, BaseTokenizerFields, Tokenizer};
+use crate::config::TokenizerConfig;
+
+pub struct WhitespaceTokenizer {
+    base: BaseTokenizerFields,
+}
+
+impl WhitespaceTokenizer {
+    pub fn new(config: TokenizerConfig) -> Self {
+        Self {
+            base: BaseTokenizerFields::new(config),
+        }
+    }
+}
+
+impl Tokenizer for WhitespaceTokenizer {
+    fn tokenize(&self, text: &str) -> Vec<String> {
+        let tokens: Vec<String> = text
+            .split_whitespace()
+            .filter(|s| !s.is_empty())
+            .map(|s| s.to_string())
+            .collect();
+
+        if self.base.has_preserve_patterns() {
+            apply_preserve_patterns(tokens, self.base.preserve_patterns(), text, &self.base.config)
+        } else {
+            post_process(tokens, &self.base.config)
+        }
+    }
+
+}

data/lib/tokenkit/config.rb

@@ -0,0 +1,74 @@
+module TokenKit
+  class Config
+    attr_accessor :strategy, :regex, :grapheme_extended, :min_gram, :max_gram, :delimiter, :split_on_chars, :lowercase, :remove_punctuation, :preserve_patterns
+
+    def self.instance
+      @instance ||= new
+    end
+
+    def initialize
+      @strategy = :unicode
+      @lowercase = true
+      @remove_punctuation = false
+      @preserve_patterns = []
+      @grapheme_extended = true
+      @min_gram = 2
+      @max_gram = 10
+      @delimiter = "/"
+      @split_on_chars = " \t\n\r"
+    end
+
+    def apply!
+      config_hash = {
+        "strategy" => strategy.to_s,
+        "lowercase" => lowercase,
+        "remove_punctuation" => remove_punctuation,
+        "preserve_patterns" => preserve_patterns.map { |p| pattern_to_string(p) }
+      }
+
+      if strategy == :pattern && regex
+        config_hash["regex"] = regex
+      end
+
+      if strategy == :grapheme
+        config_hash["extended"] = grapheme_extended
+      end
+
+      if strategy == :edge_ngram || strategy == :ngram
+        config_hash["min_gram"] = min_gram
+        config_hash["max_gram"] = max_gram
+      end
+
+      if strategy == :path_hierarchy
+        config_hash["delimiter"] = delimiter
+      end
+
+      if strategy == :char_group
+        config_hash["split_on_chars"] = split_on_chars
+      end
+
+      TokenKit.configure(config_hash)
+    end
+
+    def to_h
+      {
+        strategy: strategy,
+        regex: regex,
+        grapheme_extended: grapheme_extended,
+        min_gram: min_gram,
+        max_gram: max_gram,
+        delimiter: delimiter,
+        split_on_chars: split_on_chars,
+        lowercase: lowercase,
+        remove_punctuation: remove_punctuation,
+        preserve_patterns: preserve_patterns
+      }.compact
+    end
+
+    private
+
+    def pattern_to_string(pattern)
+      pattern.is_a?(Regexp) ? pattern.source : pattern.to_s
+    end
+  end
+end

data/lib/tokenkit/config_builder.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+require_relative 'regex_converter'
+
+module TokenKit
+  # Builder for creating immutable Configuration objects
+  class ConfigBuilder
+    attr_accessor :strategy, :lowercase, :remove_punctuation, :preserve_patterns
+    attr_accessor :regex, :grapheme_extended, :min_gram, :max_gram
+    attr_accessor :delimiter, :split_on_chars
+
+    # Default values
+    DEFAULTS = {
+      strategy: :unicode,
+      lowercase: true,
+      remove_punctuation: false,
+      preserve_patterns: [],
+      grapheme_extended: true,
+      min_gram: 2,
+      max_gram: 10,
+      delimiter: "/",
+      split_on_chars: " \t\n\r"
+    }.freeze
+
+    VALID_STRATEGIES = [
+      :unicode, :whitespace, :pattern, :sentence, :grapheme, :keyword,
+      :edge_ngram, :ngram, :path_hierarchy, :url_email, :char_group,
+      :letter, :lowercase
+    ].freeze
+
+    def initialize(base_config = nil)
+      if base_config
+        # Copy from existing config
+        @strategy = base_config.strategy
+        @lowercase = base_config.lowercase
+        @remove_punctuation = base_config.remove_punctuation
+        @preserve_patterns = base_config.preserve_patterns.dup
+        @regex = base_config.instance_variable_get(:@regex) if base_config.instance_variable_defined?(:@regex)
+        @grapheme_extended = base_config.instance_variable_get(:@grapheme_extended) || DEFAULTS[:grapheme_extended]
+        @min_gram = base_config.instance_variable_get(:@min_gram) || DEFAULTS[:min_gram]
+        @max_gram = base_config.instance_variable_get(:@max_gram) || DEFAULTS[:max_gram]
+        @delimiter = base_config.instance_variable_get(:@delimiter) || DEFAULTS[:delimiter]
+        @split_on_chars = base_config.instance_variable_get(:@split_on_chars) || DEFAULTS[:split_on_chars]
+      else
+        # Start with defaults
+        DEFAULTS.each do |key, value|
+          instance_variable_set("@#{key}", value.is_a?(Array) ? value.dup : value)
+        end
+      end
+    end
+
+    # Build an immutable Configuration object
+    # @return [Configuration] The built configuration
+    # @raise [Error] if configuration is invalid
+    def build
+      validate!
+
+      config_hash = build_config_hash
+      Configuration.new(config_hash, self)
+    end
+
+    private
+
+    def validate!
+      # Validate strategy
+      unless VALID_STRATEGIES.include?(@strategy)
+        raise Error, "Invalid strategy: #{@strategy}. Valid strategies are: #{VALID_STRATEGIES.join(', ')}"
+      end
+
+      # Strategy-specific validations
+      case @strategy
+      when :pattern
+        raise Error, "Pattern strategy requires a regex" unless @regex
+        if @regex.is_a?(String)
+          RegexConverter.validate!(@regex)
+        end
+      when :edge_ngram, :ngram
+        raise Error, "min_gram must be positive, got #{@min_gram}" if @min_gram < 1
+        raise Error, "max_gram (#{@max_gram}) must be >= min_gram (#{@min_gram})" if @max_gram < @min_gram
+      when :path_hierarchy
+        raise Error, "Path hierarchy requires a delimiter" if @delimiter.nil? || @delimiter.empty?
+      when :lowercase
+        # Warn if lowercase: false with :lowercase strategy
+        if !@lowercase
+          warn "Warning: The :lowercase strategy always lowercases text. The 'lowercase: false' setting will be ignored."
+        end
+      end
+    end
+
+    def build_config_hash
+      config = {
+        "strategy" => @strategy.to_s,
+        "lowercase" => @lowercase,
+        "remove_punctuation" => @remove_punctuation,
+        "preserve_patterns" => RegexConverter.patterns_to_rust(@preserve_patterns)
+      }
+
+      # Add strategy-specific parameters
+      case @strategy
+      when :pattern
+        config["regex"] = @regex.is_a?(Regexp) ? RegexConverter.to_rust(@regex) : @regex.to_s
+      when :grapheme
+        config["extended"] = @grapheme_extended
+      when :edge_ngram, :ngram
+        config["min_gram"] = @min_gram
+        config["max_gram"] = @max_gram
+      when :path_hierarchy
+        config["delimiter"] = @delimiter
+      when :char_group
+        config["split_on_chars"] = @split_on_chars
+      end
+
+      config
+    end
+  end
+
+  # Immutable configuration object
+  class Configuration
+    attr_reader :strategy, :lowercase, :remove_punctuation, :preserve_patterns
+    attr_reader :regex, :grapheme_extended, :min_gram, :max_gram, :delimiter, :split_on_chars
+
+    def initialize(config_hash, builder = nil)
+      @strategy = config_hash["strategy"]&.to_sym || :unicode
+      @lowercase = config_hash.fetch("lowercase", true)
+      @remove_punctuation = config_hash.fetch("remove_punctuation", false)
+      @raw_hash = config_hash.freeze
+
+      # Store builder data for creating new builders from this config
+      if builder
+        # Store original Ruby patterns, not the converted strings
+        @preserve_patterns = builder.preserve_patterns.freeze
+        @regex = builder.regex
+        @grapheme_extended = builder.grapheme_extended
+        @min_gram = builder.min_gram
+        @max_gram = builder.max_gram
+        @delimiter = builder.delimiter
+        @split_on_chars = builder.split_on_chars
+      else
+        # Extract from raw_hash for backward compatibility
+        @preserve_patterns = config_hash.fetch("preserve_patterns", []).freeze
+        @regex = config_hash["regex"]
+        @grapheme_extended = config_hash.fetch("extended", ConfigBuilder::DEFAULTS[:grapheme_extended])
+        @min_gram = config_hash.fetch("min_gram", ConfigBuilder::DEFAULTS[:min_gram])
+        @max_gram = config_hash.fetch("max_gram", ConfigBuilder::DEFAULTS[:max_gram])
+        @delimiter = config_hash.fetch("delimiter", ConfigBuilder::DEFAULTS[:delimiter])
+        @split_on_chars = config_hash.fetch("split_on_chars", ConfigBuilder::DEFAULTS[:split_on_chars])
+      end
+    end
+
+    # Create a new builder initialized with this configuration
+    def to_builder
+      ConfigBuilder.new(self)
+    end
+
+    # Strategy-specific accessors
+    def pattern?
+      strategy == :pattern
+    end
+
+    def grapheme?
+      strategy == :grapheme
+    end
+
+    def extended
+      @grapheme_extended
+    end
+
+    def edge_ngram?
+      strategy == :edge_ngram
+    end
+
+    def ngram?
+      strategy == :ngram
+    end
+
+    def path_hierarchy?
+      strategy == :path_hierarchy
+    end
+
+    def char_group?
+      strategy == :char_group
+    end
+
+    def letter?
+      strategy == :letter
+    end
+
+    def lowercase?
+      strategy == :lowercase
+    end
+
+    def to_h
+      @raw_hash.dup
+    end
+
+    def to_rust_config
+      @raw_hash
+    end
+
+    def inspect
+      "#<TokenKit::Configuration strategy=#{strategy} lowercase=#{lowercase} remove_punctuation=#{remove_punctuation}>"
+    end
+
+    # Check equality with another configuration
+    def ==(other)
+      other.is_a?(Configuration) && to_h == other.to_h
+    end
+  end
+end

data/lib/tokenkit/config_compat.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module TokenKit
+  # Compatibility wrapper that mimics the old Config singleton API
+  # This allows us to migrate gradually
+  class Config
+    # Singleton pattern for backward compatibility
+    def self.instance
+      @instance ||= new
+    end
+
+    # Delegate all accessors to the global config builder
+    def method_missing(method, *args, &block)
+      if method.to_s.end_with?('=')
+        # Setter - store in temporary builder
+        attr = method.to_s.chomp('=').to_sym
+        @temp_builder ||= TokenKit.config_hash.to_builder
+        @temp_builder.send(method, *args, &block) if @temp_builder.respond_to?(method)
+      else
+        # Getter - get from current config or temp builder
+        if @temp_builder && @temp_builder.respond_to?(method)
+          @temp_builder.send(method)
+        else
+          TokenKit.config_hash.send(method) if TokenKit.config_hash.respond_to?(method)
+        end
+      end
+    end
+
+    def respond_to_missing?(method, include_private = false)
+      # Avoid infinite recursion by checking config_hash instead of config
+      return true if [:strategy=, :lowercase=, :remove_punctuation=, :preserve_patterns=,
+                      :regex=, :grapheme_extended=, :min_gram=, :max_gram=,
+                      :delimiter=, :split_on_chars=,
+                      :strategy, :lowercase, :remove_punctuation, :preserve_patterns,
+                      :regex, :grapheme_extended, :min_gram, :max_gram,
+                      :delimiter, :split_on_chars].include?(method)
+      super
+    end
+
+    # Called by TokenKit.configure to get the built config
+    def build_config
+      builder = @temp_builder || TokenKit.config_hash.to_builder
+      @temp_builder = nil  # Clear after building
+      builder
+    end
+
+    # Reset temporary builder
+    def reset_temp
+      @temp_builder = nil
+    end
+  end
+end

data/lib/tokenkit/configuration.rb

@@ -0,0 +1,194 @@
+module TokenKit
+  # Immutable configuration object representing tokenizer settings.
+  #
+  # This class provides read-only access to configuration values and
+  # convenient predicate methods for checking the current strategy.
+  #
+  # @example Access configuration
+  #   config = TokenKit.config_hash
+  #   config.strategy           # => :unicode
+  #   config.lowercase          # => true
+  #   config.preserve_patterns  # => [/\d+mg/i]
+  #
+  # @example Check strategy type
+  #   config.unicode?           # => true
+  #   config.edge_ngram?        # => false
+  #
+  class Configuration
+    # @return [Symbol] The tokenization strategy
+    attr_reader :strategy
+
+    # @return [Boolean] Whether to lowercase tokens
+    attr_reader :lowercase
+
+    # @return [Boolean] Whether to remove punctuation
+    attr_reader :remove_punctuation
+
+    # @return [Array<Regexp>] Patterns to preserve from modification
+    attr_reader :preserve_patterns
+
+    # Creates a new configuration from a hash.
+    #
+    # @param config_hash [Hash] Configuration values from Rust
+    # @api private
+    #
+    def initialize(config_hash)
+      @strategy = config_hash["strategy"]&.to_sym || :unicode
+      @lowercase = config_hash.fetch("lowercase", true)
+      @remove_punctuation = config_hash.fetch("remove_punctuation", false)
+      @preserve_patterns = config_hash.fetch("preserve_patterns", []).freeze
+      @raw_hash = config_hash
+    end
+
+    # @return [Boolean] true if using pattern tokenization strategy
+    def pattern?
+      strategy == :pattern
+    end
+
+    # @return [String, nil] The regex pattern for pattern strategy
+    def regex
+      @raw_hash["regex"]
+    end
+
+    # @return [Boolean] true if using grapheme tokenization strategy
+    def grapheme?
+      strategy == :grapheme
+    end
+
+    # @return [Boolean, nil] Whether to use extended grapheme clusters
+    def extended
+      @raw_hash["extended"]
+    end
+
+    # @return [Boolean] true if using edge n-gram tokenization strategy
+    def edge_ngram?
+      strategy == :edge_ngram
+    end
+
+    # @return [Integer, nil] Minimum n-gram size for n-gram strategies
+    def min_gram
+      @raw_hash["min_gram"]
+    end
+
+    # @return [Integer, nil] Maximum n-gram size for n-gram strategies
+    def max_gram
+      @raw_hash["max_gram"]
+    end
+
+    # @return [Boolean] true if using path hierarchy tokenization strategy
+    def path_hierarchy?
+      strategy == :path_hierarchy
+    end
+
+    # @return [String, nil] Delimiter for path hierarchy strategy
+    def delimiter
+      @raw_hash["delimiter"]
+    end
+
+    # @return [Boolean] true if using n-gram tokenization strategy
+    def ngram?
+      strategy == :ngram
+    end
+
+    # @return [Boolean] true if using character group tokenization strategy
+    def char_group?
+      strategy == :char_group
+    end
+
+    # @return [String, nil] Characters to split on for char_group strategy
+    def split_on_chars
+      @raw_hash["split_on_chars"]
+    end
+
+    # @return [Boolean] true if using letter tokenization strategy
+    def letter?
+      strategy == :letter
+    end
+
+    # @return [Boolean] true if using lowercase tokenization strategy
+    def lowercase?
+      strategy == :lowercase
+    end
+
+    # @return [Boolean] true if using unicode tokenization strategy
+    def unicode?
+      strategy == :unicode
+    end
+
+    # @return [Boolean] true if using whitespace tokenization strategy
+    def whitespace?
+      strategy == :whitespace
+    end
+
+    # @return [Boolean] true if using sentence tokenization strategy
+    def sentence?
+      strategy == :sentence
+    end
+
+    # @return [Boolean] true if using keyword tokenization strategy
+    def keyword?
+      strategy == :keyword
+    end
+
+    # @return [Boolean] true if using url_email tokenization strategy
+    def url_email?
+      strategy == :url_email
+    end
+
+    # Converts configuration to a hash.
+    #
+    # @return [Hash] Configuration as a hash
+    #
+    # @example
+    #   config.to_h
+    #   # => {"strategy" => "unicode", "lowercase" => true, ...}
+    #
+    def to_h
+      @raw_hash.dup
+    end
+
+    # Returns a string representation of the configuration.
+    #
+    # @return [String] Human-readable configuration summary
+    #
+    def inspect
+      "#<TokenKit::Configuration strategy=#{strategy} lowercase=#{lowercase} remove_punctuation=#{remove_punctuation}>"
+    end
+
+    # Converts configuration to format expected by Rust.
+    #
+    # @return [Hash] Configuration hash for Rust FFI
+    # @api private
+    #
+    def to_rust_config
+      @raw_hash
+    end
+
+    # Creates a ConfigBuilder from this configuration for modification.
+    #
+    # @return [ConfigBuilder] A builder initialized with this configuration
+    #
+    # @example
+    #   builder = config.to_builder
+    #   builder.lowercase = false
+    #   new_config = builder.build
+    #
+    def to_builder
+      builder = ConfigBuilder.new
+      builder.strategy = strategy
+      builder.lowercase = lowercase
+      builder.remove_punctuation = remove_punctuation
+      builder.preserve_patterns = preserve_patterns.dup
+
+      # Copy strategy-specific settings
+      builder.regex = regex if pattern?
+      builder.extended = extended if grapheme?
+      builder.min_gram = min_gram if edge_ngram? || ngram?
+      builder.max_gram = max_gram if edge_ngram? || ngram?
+      builder.delimiter = delimiter if path_hierarchy?
+      builder.split_on_chars = split_on_chars if char_group?
+
+      builder
+    end
+  end
+end