kotoshu 0.3.0

data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb

@@ -0,0 +1,316 @@
+# frozen_string_literal: true
+
+require_relative 'base_strategy'
+require_relative '../suggestion'
+require_relative '../suggestion_set'
+require_relative '../../embeddings'
+
+module Kotoshu
+  module Suggestions
+    module Strategies
+      # Semantic strategy using FastText ONNX embeddings.
+      #
+      # Provides embedding-based spell correction for:
+      # - Typos: Re-ranks edit-distance candidates by semantic similarity
+      # - Real-word errors: Detects when valid words are used incorrectly in context
+      #
+      # This strategy works alongside other strategies (EditDistance, Phonetic, etc.)
+      # to provide comprehensive spell checking with semantic awareness.
+      #
+      # @example Basic usage
+      #   strategy = SemanticStrategy.new(language_code: 'en')
+      #   suggestions = strategy.generate(context)
+      #
+      # @example With preloaded embeddings (faster)
+      #   strategy = SemanticStrategy.new(
+      #     language_code: 'en',
+      #     preload_embeddings: true
+      #   )
+      #   suggestions = strategy.generate(context)
+      class SemanticStrategy < BaseStrategy
+        # @return [String] Language code (ISO 639-1)
+        attr_reader :language_code
+
+        # @return [Embeddings::Vocabulary] The vocabulary
+        attr_reader :vocabulary
+
+        # @return [Embeddings::OnnxRuntimeModel] The ONNX model
+        attr_reader :model
+
+        # @return [Embeddings::SimilaritySearch] The similarity search
+        attr_reader :search
+
+        # Create a new semantic strategy.
+        #
+        # @param language_code [String] ISO 639-1 language code
+        # @param cache [Cache::ModelCache, nil] Optional cache instance
+        # @param preload_embeddings [Boolean] Whether to preload embeddings
+        # @param max_context_window [Integer] Words to consider for context
+        # @param min_semantic_similarity [Float] Minimum similarity for semantic suggestions
+        # @param semantic_boost_weight [Float] Weight for semantic similarity in re-ranking
+        # @param config [Hash] Additional configuration
+        def initialize(language_code:, cache: nil, preload_embeddings: false,
+                       max_context_window: 5, min_semantic_similarity: 0.5,
+                       semantic_boost_weight: 0.3, **config)
+          super(name: :semantic, **config)
+          @language_code = language_code
+          @max_context_window = max_context_window
+          @min_semantic_similarity = min_semantic_similarity
+          @semantic_boost_weight = semantic_boost_weight
+
+          # Initialize embedding components
+          initialize_embeddings(cache, preload_embeddings)
+        end
+
+        # Generate suggestions using semantic similarity.
+        #
+        # Handles two cases:
+        # 1. Word not in vocabulary (typo): Re-ranks edit-distance candidates
+        # 2. Word in vocabulary (real-word error): Finds semantically similar alternatives
+        #
+        # @param context [Context] The suggestion context
+        # @return [SuggestionSet] Generated suggestions
+        def generate(context)
+          word = context.word
+          max_results = context.max_results || max_results
+
+          # Ensure embeddings are loaded
+          return SuggestionSet.empty unless @search
+
+          # Case 1: Word not in vocabulary (typo)
+          unless @vocabulary.include?(word)
+            return generate_for_typo(context)
+          end
+
+          # Case 2: Real-word error detection
+          # Find semantically similar words that might be correct in context
+          generate_for_real_word_error(context)
+        end
+
+        # Check if this strategy should handle the context.
+        #
+        # Semantic strategy handles:
+        # - Words not in vocabulary (for typo re-ranking)
+        # - Words in vocabulary (for real-word error detection)
+        #
+        # @param context [Context] The suggestion context
+        # @return [Boolean] True if the strategy should handle this context
+        def handles?(context)
+          return false unless enabled?
+          return false unless @search && @vocabulary
+
+          # Handle all words - we filter in generate()
+          true
+        end
+
+        # Get embedding for a word.
+        #
+        # @param word [String] The word
+        # @return [Array<Float>, nil] Embedding vector or nil if not found
+        def embedding_for(word)
+          return nil unless @search
+
+          @search.send(:get_embedding, word)
+        end
+
+        # Compute semantic similarity between two words.
+        #
+        # @param word1 [String] First word
+        # @param word2 [String] Second word
+        # @return [Float, nil] Cosine similarity or nil if either word not found
+        def semantic_similarity(word1, word2)
+          return nil unless @search
+
+          @search.similarity(word1, word2)
+        end
+
+        # Find semantically similar words.
+        #
+        # @param word [String] The query word
+        # @param k [Integer] Number of neighbors
+        # @return [Array<Hash>] Array of {word, similarity} hashes
+        def find_similar_words(word, k: 10)
+          return [] unless @search
+
+          @search.find_nearest(word, k: k, exclude_self: false)
+        end
+
+        # String representation.
+        #
+        # @return [String] String representation
+        def to_s
+          "SemanticStrategy(language: #{@language_code}, vocab_size: #{@vocabulary&.size || 0}, loaded: #{@search && true})"
+        end
+        alias inspect to_s
+
+        private
+
+        # Initialize embedding components.
+        #
+        # @param cache [Cache::ModelCache, nil] Cache instance
+        # @param preload [Boolean] Whether to preload embeddings
+        def initialize_embeddings(cache, preload)
+          # Try to load from cache
+          @search = Embeddings::SimilaritySearch.from_cache(
+            @language_code,
+            cache: cache,
+            preload: preload
+          )
+
+          # Extract vocabulary and model from search
+          if @search
+            @vocabulary = @search.vocabulary
+            @model = @search.model
+          else
+            @vocabulary = nil
+            @model = nil
+
+            warn "Warning: Could not load ONNX model for language '#{@language_code}'. Semantic strategy will be disabled." if $VERBOSE
+          end
+        end
+
+        # Generate suggestions for a typo (word not in vocabulary).
+        #
+        # Uses semantic similarity to re-rank candidates from other strategies.
+        #
+        # @param context [Context] The suggestion context
+        # @return [SuggestionSet] Re-ranked suggestions
+        def generate_for_typo(context)
+          word = context.word
+          max_results = context.max_results || max_results
+
+          # For typos, we find semantically similar words in vocabulary
+          # that are also close in spelling (handled by edit distance strategy)
+          neighbors = @search.find_nearest(
+            word,
+            k: max_results * 2,  # Get more candidates for filtering
+            exclude_self: true,
+            min_similarity: @min_semantic_similarity
+          )
+
+          return SuggestionSet.empty if neighbors.empty?
+
+          # Convert to suggestions
+          # Confidence is based on semantic similarity
+          suggestions = neighbors.map do |neighbor|
+            similarity = neighbor[:similarity]
+            confidence = normalize_similarity(similarity)
+
+            # Calculate "distance" as inverse of similarity
+            # High similarity = low distance
+            distance = similarity_to_distance(similarity)
+
+            create_suggestion(
+              neighbor[:word],
+              distance: distance,
+              confidence: confidence,
+              semantic_similarity: similarity
+            )
+          end
+
+          # Sort and limit
+          SuggestionSet.new(suggestions, max_size: max_results)
+        end
+
+        # Generate suggestions for a real-word error.
+        #
+        # Finds semantically similar words that might be correct in context.
+        #
+        # @param context [Context] The suggestion context
+        # @return [SuggestionSet] Alternative suggestions
+        def generate_for_real_word_error(context)
+          word = context.word
+          max_results = context.max_results || max_results
+
+          # Get context words from the surrounding text
+          context_words = get_context_words(context)
+
+          # Find semantically similar words
+          neighbors = @search.find_nearest(
+            word,
+            k: max_results * 3,
+            exclude_self: true,
+            min_similarity: @min_semantic_similarity
+          )
+
+          return SuggestionSet.empty if neighbors.empty?
+
+          # Re-rank by context similarity
+          suggestions = neighbors.map do |neighbor|
+            candidate_word = neighbor[:word]
+            similarity = neighbor[:similarity]
+
+            # Check if candidate makes more sense in context
+            context_score = compute_context_fit(candidate_word, context_words)
+
+            # Combine semantic similarity with context fit
+            combined_score = (similarity * 0.7) + (context_score * 0.3)
+
+            confidence = normalize_similarity(combined_score)
+            distance = similarity_to_distance(combined_score)
+
+            create_suggestion(
+              candidate_word,
+              distance: distance,
+              confidence: confidence,
+              semantic_similarity: similarity,
+              context_score: context_score
+            )
+          end
+
+          # Sort by combined score and limit
+          SuggestionSet.new(suggestions.sort_by { |s| -s.metadata[:context_score] }, max_size: max_results)
+        end
+
+        # Get context words for semantic analysis.
+        #
+        # @param context [Context] The suggestion context
+        # @return [Array<String>] Context words
+        def get_context_words(context)
+          # For now, return empty - context analysis would need full text
+          # This could be extended in the future
+          []
+        end
+
+        # Compute how well a word fits in context.
+        #
+        # @param candidate [String] Candidate word
+        # @param context_words [Array<String>] Context words
+        # @return [Float] Context fit score (0.0 to 1.0)
+        def compute_context_fit(candidate, context_words)
+          return 0.5 if context_words.empty?
+
+          # Compute average similarity between candidate and context words
+          similarities = context_words.map do |ctx_word|
+            @search.similarity(candidate, ctx_word)
+          end.compact
+
+          return 0.5 if similarities.empty?
+
+          similarities.sum / similarities.size
+        end
+
+        # Normalize similarity to confidence (0.0 to 1.0).
+        #
+        # @param similarity [Float] Cosine similarity (-1.0 to 1.0)
+        # @return [Float] Normalized confidence (0.0 to 1.0)
+        def normalize_similarity(similarity)
+          # Map from [-1, 1] to [0, 1]
+          ((similarity + 1) / 2.0).clamp(0.0, 1.0)
+        end
+
+        # Convert similarity to "distance" for ranking.
+        #
+        # @param similarity [Float] Cosine similarity (-1.0 to 1.0)
+        # @return [Integer] Pseudo-distance (lower = better)
+        def similarity_to_distance(similarity)
+          # Map similarity to distance: higher similarity = lower distance
+          # Similarity 1.0 -> distance 0
+          # Similarity 0.0 -> distance 2
+          # Similarity -1.0 -> distance 4
+          ((1.0 - similarity) * 2).to_i.clamp(0, 5)
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Suggestions
+    module Strategies
+      # SymSpell suggestion strategy.
+      #
+      # Uses deletion distance algorithm for fast approximate string matching.
+      # Pre-computes deletion variants for all dictionary words, enabling O(1)
+      # lookup for common misspellings.
+      #
+      # This is 10-100x faster than EditDistanceStrategy for large dictionaries.
+      #
+      # The algorithm works by:
+      # 1. Pre-computing single deletion variants for each dictionary word
+      # 2. Looking up input word's deletion variants in the pre-computed map
+      # 3. Distance is inferred from the deletion level
+      #
+      # @see https://github.com/wolfgarbe/SymSpell Original SymSpell paper
+      class SymSpellStrategy < BaseStrategy
+        # Maximum deletion distance to consider
+        DEFAULT_MAX_DELETION_DISTANCE = 2
+        # Maximum dictionary words to process (increased for better coverage)
+        DEFAULT_MAX_DICTIONARY_SIZE = 500_000
+        # Enable transposition handling (slower pre-computation, better accuracy)
+        DEFAULT_HANDLE_TRANSPOSITIONS = true
+
+        # Create a new SymSpell strategy.
+        #
+        # @param dictionary [Object] Dictionary to use for suggestions
+        # @param name [String, Symbol] Strategy name
+        # @param config [Hash] Configuration options
+        # @option config [Integer] max_deletion_distance Maximum deletion distance (default: 2)
+        # @option config [Integer] max_results Maximum results to return (default: 10)
+        # @option config [Integer] max_dictionary_size Maximum words to process (default: 500_000)
+        # @option config [Boolean] handle_transpositions Generate transposition variants (default: true)
+        def initialize(dictionary:, name: :symspell, **config)
+          super(name: name, **config)
+          @dictionary = dictionary
+          @max_deletion_distance = config.fetch(:max_deletion_distance, DEFAULT_MAX_DELETION_DISTANCE)
+          @max_dictionary_size = config.fetch(:max_dictionary_size, DEFAULT_MAX_DICTIONARY_SIZE)
+          @handle_transpositions = config.fetch(:handle_transpositions, DEFAULT_HANDLE_TRANSPOSITIONS)
+          @deletes = Hash.new { |h, k| h[k] = [] } # deletion_variant -> [original_words]
+          @words = Set.new
+          precompute!
+        end
+
+        # Generate suggestions using deletion distance.
+        #
+        # @param context [Context] The suggestion context
+        # @return [SuggestionSet] Generated suggestions
+        def generate(context)
+          word = context.word
+          max_dist = get_config(:max_deletion_distance, @max_deletion_distance)
+
+          # Normalize to lowercase for case-insensitive matching
+          word_lower = word.downcase
+
+          # Check if word is in dictionary
+          return SuggestionSet.empty if @words.include?(word_lower)
+
+          # Collect candidates with their distances
+          candidates = {}
+          checked = Set.new([word_lower])
+
+          # First, check if the input word is a deletion variant of any dictionary word
+          @deletes[word_lower].each do |dict_word|
+            candidates[dict_word] ||= 1
+          end
+
+          # If transpositions are enabled, check them too
+          if @handle_transpositions
+            generate_transpositions(word_lower).each do |transposed|
+              @deletes[transposed].each do |dict_word|
+                candidates[dict_word] ||= 1
+              end
+            end
+          end
+
+          # Generate deletion variants and check for matches
+          max_dist.times do |dist|
+            generate_deletions_from_set(checked).each do |variant|
+              next if checked.include?(variant)
+
+              checked.add(variant)
+
+              # Check if variant is directly in dictionary
+              candidates[variant] = dist + 1 if @words.include?(variant)
+
+              # Check if variant maps to dictionary words
+              @deletes[variant].each do |dict_word|
+                # Distance = deletions from input + deletions from dict_word
+                # Both reach the same variant
+                candidates[dict_word] ||= dist + 2
+              end
+            end
+          end
+
+          # Sort by distance and create suggestions
+          sorted_words = candidates.sort_by { |_, dist| dist }.map(&:first)
+          create_suggestion_set(sorted_words, distances: candidates, original_word: context.word)
+        end
+
+        # Pre-compute deletion variants for all dictionary words.
+        #
+        # This is called during initialization and builds the index.
+        def precompute!
+          words = dictionary_words(@dictionary)
+
+          words.first(@max_dictionary_size).each do |word|
+            next if word.nil? || word.empty?
+
+            word_lower = word.downcase
+            @words.add(word_lower)
+
+            # Generate only single deletion variants for efficiency
+            # Multiple deletions are handled during lookup
+            generate_single_deletions(word_lower).each do |variant|
+              @deletes[variant] << word_lower
+            end
+
+            # Generate transposition variants if enabled
+            if @handle_transpositions
+              generate_transpositions(word_lower).each do |variant|
+                @deletes[variant] << word_lower
+              end
+            end
+          end
+        end
+
+        # Generate all adjacent transposition variants of a word.
+        #
+        # For example, "world" → ["owrld", "wrold", "wolrd", "wordl"]
+        #
+        # @param word [String] The word
+        # @return [Array<String>] Array of variants with adjacent characters swapped
+        def generate_transpositions(word)
+          variants = []
+          word.chars.each_with_index do |_, i|
+            next if i == word.length - 1 # Can't swap last character
+
+            variant = word.dup
+            variant[i], variant[i + 1] = variant[i + 1], variant[i]
+            variants << variant unless variant == word
+          end
+          variants
+        end
+
+        # Calculate deletion distance between two words.
+        #
+        # For SymSpell, this is the length of their longest common subsequence
+        # based distance (minimum deletions to make them equal).
+        #
+        # @param str1 [String] First word
+        # @param str2 [String] Second word
+        # @return [Integer] Deletion distance
+        def deletion_distance(str1, str2)
+          return str2.length if str1.empty?
+          return str1.length if str2.empty?
+          return 0 if str1 == str2
+
+          # Simple approach: find if one can be transformed to the other
+          # by only deletions (check if str1 is subsequence of str2 or vice versa)
+          if is_subsequence?(str1, str2)
+            str2.length - str1.length
+          elsif is_subsequence?(str2, str1)
+            str1.length - str2.length
+          else
+            # Fallback to edit distance approximation
+            # This shouldn't happen often with proper SymSpell usage
+            lcs_len = longest_common_subsequence_length(str1, str2)
+            str1.length + str2.length - 2 * lcs_len
+          end
+        end
+
+        private
+
+        # Generate all single-deletion variants of a word.
+        #
+        # @param word [String] The word
+        # @return [Array<String>] Array of variants with one character deleted
+        def generate_single_deletions(word)
+          variants = []
+          word.chars.each_with_index do |_, i|
+            variant = word[0...i] + word[(i + 1)..].to_s
+            variants << variant unless variant.empty? || variant == word
+          end
+          variants
+        end
+
+        # Generate deletion variants from a set of words.
+        #
+        # @param words_set [Set<String>] Set of words to process
+        # @return [Set<String>] New set with all single deletions
+        def generate_deletions_from_set(words_set)
+          result = Set.new
+          words_set.each do |word|
+            generate_single_deletions(word).each do |variant|
+              result.add(variant)
+            end
+          end
+          result
+        end
+
+        # Check if str1 is a subsequence of str2.
+        #
+        # @param str1 [String] Potential subsequence
+        # @param str2 [String] String to check against
+        # @return [Boolean] True if str1 is subsequence of str2
+        def is_subsequence?(str1, str2)
+          return true if str1.empty?
+          return false if str1.length > str2.length
+
+          i = 0
+          str2.each_char do |c|
+            i += 1 if c == str1[i]
+            return true if i == str1.length
+          end
+          i == str1.length
+        end
+
+        # Calculate the length of the longest common subsequence.
+        #
+        # Uses dynamic programming for efficiency.
+        #
+        # @param str1 [String] First string
+        # @param str2 [String] Second string
+        # @return [Integer] LCS length
+        def longest_common_subsequence_length(str1, str2)
+          return 0 if str1.empty? || str2.empty?
+
+          # Use shorter string for inner loop
+          str1, str2 = str2, str1 if str1.length > str2.length
+
+          # Previous row of DP table
+          previous = Array.new(str1.length + 1, 0)
+
+          str2.each_char do |char2|
+            current = [0] # First column is always 0
+
+            str1.each_char.with_index do |char1, i|
+              current << if char1 == char2
+                           previous[i] + 1
+                         else
+                           [current[i], previous[i + 1]].max
+                         end
+            end
+
+            previous = current
+          end
+
+          previous.last
+        end
+
+        # Get all words from the dictionary.
+        #
+        # @param dictionary [Object] Dictionary object
+        # @return [Array<String>] All words
+        def dictionary_words(dictionary)
+          if dictionary.respond_to?(:words)
+            dictionary.words
+          elsif dictionary.is_a?(Array)
+            dictionary
+          elsif dictionary.is_a?(Hash)
+            dictionary.keys
+          elsif dictionary.respond_to?(:all_words)
+            dictionary.all_words
+          else
+            []
+          end
+        end
+      end
+    end
+  end
+end