kotoshu 0.3.0

data/lib/kotoshu/algorithms/ngram_suggest.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Algorithms
+    # N-gram based suggestion algorithm.
+    #
+    # Ported from Spylls (Python) ngram_suggest.py
+    #
+    # This is the core Hunspell suggestion algorithm that uses n-gram
+    # similarity to rank and filter spelling corrections.
+    #
+    # The algorithm works in three stages:
+    # 1. root_score: Quick n-gram score + left common substring
+    # 2. rough_affix_score: Affixed form n-gram score
+    # 3. precise_affix_score: Full scoring with LCS, bigrams, etc.
+    module NgramSuggest
+      # Maximum number of root words to consider in first pass
+      MAX_ROOTS = 100
+
+      # Maximum number of suggestions to generate
+      MAX_GUESSES = 200
+
+      class << self
+        # Main entry point for n-gram based suggestions.
+        #
+        # @param misspelling [String] The misspelled word
+        # @param dictionary_words [Array<Hash>] Dictionary entries with stem and flags
+        # @param prefixes [Hash] Prefix flags to prefix objects mapping
+        # @param suffixes [Hash] Suffix flags to suffix objects mapping
+        # @param known [Set<String>] Already suggested words (to avoid duplicates)
+        # @param maxdiff [Integer] MAXDIFF value from aff file (0-10)
+        # @param onlymaxdiff [Boolean] ONLYMAXDIFF flag
+        # @param has_phonetic [Boolean] Whether PHONE table exists in aff file
+        # @yield [String] Each suggestion
+        #
+        # This is a simplified version that works with basic dictionary structures.
+        # Full implementation would need affix flag parsing and Word model objects.
+        def suggest(misspelling,
+                    dictionary_words:,
+                    prefixes: {},
+                    suffixes: {},
+                    known: Set.new,
+                    maxdiff: 2,
+                    onlymaxdiff: true,
+                    has_phonetic: false,
+                    &block)
+
+          # Stage 1: Find best root candidates by n-gram score
+          root_scores = []
+
+          dictionary_words.each do |word_entry|
+            stem = word_entry[:stem] || word_entry
+
+            # Skip words with length difference > 4
+            next if (stem.length - misspelling.length).abs > 4
+
+            score = root_score(misspelling, stem)
+
+            # Use heap to keep only MAX_ROOTS best results
+            if root_scores.size >= MAX_ROOTS
+              # Keep only the best scores
+              root_scores = root_scores.sort.reverse.first(MAX_ROOTS)
+            end
+
+            root_scores << [score, word_entry] if score > 0
+          end
+
+          # Stage 2: Generate affixed forms and score them
+          threshold = detect_threshold(misspelling)
+          guess_scores = []
+
+          # Sort by score descending
+          root_scores.sort_by { |score, _| -score }.first(MAX_ROOTS).each do |(_, root_entry)|
+            root = root_entry[:stem] || root_entry
+
+            # Generate forms with suffixes
+            forms = forms_for(root_entry, prefixes, suffixes, similar_to: misspelling)
+
+            forms.each do |form|
+              score = rough_affix_score(misspelling, form.to_s.downcase)
+              next unless score > threshold
+
+              guess_scores << [score, form.to_s, form.to_s]
+            end
+          end
+
+          # Limit to MAX_GUESSES and sort by score
+          guesses = guess_scores.sort.reverse.first(MAX_GUESSES)
+
+          # Stage 3: Calculate precise scores
+          fact = maxdiff >= 0 ? (10.0 - maxdiff) / 5.0 : 1.0
+
+          guesses2 = guesses.map do |score, compared, real|
+            [precise_affix_score(misspelling, compared.to_s.downcase,
+                                fact, base: score, has_phonetic: has_phonetic), real.to_s]
+          end.sort.reverse
+
+          # Stage 4: Filter and yield suggestions
+          filter_guesses(guesses2, known: known, onlymaxdiff: onlymaxdiff, &block)
+        end
+
+        # Stage 1 scoring: 3-gram score + left common substring.
+        #
+        # @param word1 [String] Misspelled word
+        # @param word2 [String] Possible suggestion
+        # @return [Float] Root score
+        def root_score(word1, word2)
+          # Use lowercase for comparison as per Hunspell
+          word2_lower = word2.downcase
+
+          StringMetrics.ngram(3, word1, word2_lower, longer_worse: true) +
+            StringMetrics.leftcommonsubstring(word1, word2_lower).to_f
+        end
+
+        # Stage 2 scoring: N-gram score with n=len(word1) + left common substring.
+        #
+        # @param word1 [String] Misspelled word
+        # @param word2 [String] Possible suggestion
+        # @return [Float] Rough affix score
+        def rough_affix_score(word1, word2)
+          # Use lowercase for comparison as per Hunspell
+          word2_lower = word2.downcase
+
+          StringMetrics.ngram(word1.length, word1, word2_lower, any_mismatch: true) +
+            StringMetrics.leftcommonsubstring(word1, word2_lower).to_f
+        end
+
+        # Stage 3 scoring: Full precise scoring.
+        #
+        # Returns one of three "score groups":
+        # - > 1000: Very good (same word, different casing)
+        # - < -100: Questionable (too different)
+        # - -100 to 1000: Normal suggestion
+        #
+        # @param word1 [String] Misspelled word
+        # @param word2 [String] Possible suggestion
+        # @param diff_factor [Float] Factor based on MAXDIFF (0-2)
+        # @param base [Float] Base score from stage 2
+        # @param has_phonetic [Boolean] Whether PHONE table exists
+        # @return [Float] Precise affix score
+        def precise_affix_score(word1, word2, diff_factor, base:, has_phonetic: false)
+          # Use lowercase for LCS to catch case-only differences
+          word1_lower = word1.downcase
+          word2_lower = word2.downcase
+
+          lcs = StringMetrics.lcslen(word1_lower, word2_lower)
+
+          # Same characters with different casing -- "very good" suggestion
+          if word1.length == word2.length && word1.length == lcs
+            return base + 2000
+          end
+
+          # Score is: 2 * LCS - length difference
+          result = 2 * lcs - (word1.length - word2.length).abs
+
+          # Add common start substring length
+          result += StringMetrics.leftcommonsubstring(word1_lower, word2_lower)
+
+          # Add 1 if any characters match at same positions
+          result += 1 if StringMetrics.commoncharacters(word1_lower, word2_lower) > 0
+
+          # Add regular 4-gram score
+          result += StringMetrics.ngram(4, word1_lower, word2_lower, any_mismatch: true)
+
+          # Add weighted bigrams (both directions)
+          bigrams = (
+            StringMetrics.ngram(2, word1_lower, word2_lower, any_mismatch: true, weighted: true) +
+            StringMetrics.ngram(2, word2_lower, word1_lower, any_mismatch: true, weighted: true)
+          )
+          result += bigrams
+
+          # Apply "questionable" threshold based on diff_factor and has_phonetic
+          questionable_limit = if has_phonetic
+                                word2.length * diff_factor
+                              else
+                                (word1.length + word2.length) * diff_factor
+                              end
+
+          result -= 1000 if bigrams < questionable_limit
+
+          result
+        end
+
+        # Calculate minimum threshold for passable suggestions.
+        #
+        # Mangles the word in 3 different ways (replacing each 4th char with '*')
+        # and scores them to generate a minimum acceptable score.
+        #
+        # @param word [String] The misspelled word
+        # @return [Float] Minimum threshold score
+        def detect_threshold(word)
+          thresh = 0.0
+
+          (1..3).each do |start_pos|
+            mangled = word.chars.map.with_index do |char, pos|
+              ((pos - start_pos) % 4).zero? && pos >= start_pos ? "*" : char
+            end.join
+
+            thresh += StringMetrics.ngram(word.length, word, mangled, any_mismatch: true)
+          end
+
+          # Take average of the three scores and subtract 1
+          (thresh / 3.0) - 1
+        end
+
+        # Generate all possible affixed forms for a dictionary word.
+        #
+        # @param word_entry [Hash] Dictionary word with stem and flags
+        # @param all_prefixes [Hash] Available prefixes
+        # @param all_suffixes [Hash] Available suffixes
+        # @param similar_to [String] Original misspelling (for filtering)
+        # @return [Array<String>] Generated forms
+        def forms_for(word_entry, all_prefixes, all_suffixes, similar_to:)
+          stem = word_entry[:stem] || word_entry
+          flags = word_entry[:flags] || []
+
+          # Base form without affixes
+          res = [stem]
+
+          # Generate suffix forms
+          # Simplified: just return base form for now
+          # Full implementation would parse affix flags and apply them
+
+          res
+        end
+
+        # Filter guesses by score into quality buckets.
+        #
+        # Score buckets:
+        # - > 1000: Very good (same word, different casing)
+        # - 1000 to -100: Normal suggestions
+        # - < -100: Questionable (too different)
+        #
+        # Stops yielding when:
+        # - A very good suggestion was found and then a normal one
+        # - A questionable suggestion was found (only yields one)
+        #
+        # @param guesses [Array<Array>] Array of [score, value] pairs
+        # @param known [Set<String>] Already suggested words
+        # @param onlymaxdiff [Boolean] Whether to exclude questionable
+        # @yield [String] Each filtered suggestion
+        def filter_guesses(guesses, known:, onlymaxdiff: true)
+          seen = false
+          found = 0
+
+          guesses.each do |score, value|
+            # Stop if we saw very good and now have normal suggestions
+            return if seen && score <= 1000
+
+            if score > 1000
+              # Very good suggestion - set flag to only accept other very good ones
+              seen = true
+            elsif score < -100
+              # Questionable suggestion
+              # Stop if we already found good ones, or if we're excluding questionable
+              return if found.positive? || onlymaxdiff
+              seen = true
+            end
+
+            # Skip if this word was already suggested
+            next if known.any? { |known_word| value.include?(known_word) }
+
+            found += 1
+            yield value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/algorithms/permutations.rb

@@ -0,0 +1,283 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Algorithms
+    # Word permutation algorithms for generating spelling variations.
+    #
+    # Ported from Spylls (Python) permutations.py
+    #
+    # These functions generate various word edits that are used by the
+    # suggestion system to find possible corrections for misspelled words.
+    #
+    # Method names match Hunspell's suggest.cxx to maintain compatibility.
+    module Permutations
+      MAX_CHAR_DISTANCE = 4
+
+      module_function
+
+      # Uses REP table (typical misspellings) to replace patterns in word.
+      #
+      # If the pattern's replacement contains "_", it means replacing to " "
+      # and yielding two different hypotheses:
+      #   1. It was one (dictionary) word "foo bar" (checked as such)
+      #   2. It was words ["foo", "bar"] (checked separately)
+      #
+      # @param word [String] The word to process
+      # @param reptable [Array<Hash>] Array of replacement pattern hashes with :regexp and :replacement
+      # @yield [String, Array<String>] Each suggestion (string or array of words)
+      #
+      # @example
+      #   Kotoshu::Algorithms::Permutations.replchars("acces", [{regexp: /ac/, replacement: "ex"}]) do |sug|
+      #     puts sug
+      #   end
+      def replchars(word, reptable)
+        return if word.length < 2 || reptable.nil? || reptable.empty?
+
+        reptable.each do |pattern|
+          str = word.to_s
+          pos = 0
+
+          while (match_data = pattern[:regexp].match(str, pos))
+            suggestion = str[0...match_data.begin(0)] +
+                         pattern[:replacement].gsub('_', ' ') +
+                         str[match_data.end(0)..]
+
+            yield suggestion
+            yield suggestion.split(' ', 2) if suggestion.include?(' ')
+
+            # Move past this match to find next occurrence
+            pos = match_data.end(0)
+            break if pos >= str.length
+          end
+        end
+      end
+
+      # Uses MAP table (sets of potentially similar chars) and tries to replace them recursively.
+      #
+      # Example: Assuming MAP has entry "aáã", and we have misspelling "anarchia":
+      #   mapchars will produce: "ánarchia", "ánárchia", "ánárchiá", etc.
+      #
+      # @param word [String] The word to process
+      # @param maptable [Array<Set<String>>] Array of character sets for mapping
+      # @yield [String] Each variant with mapped characters
+      #
+      # @example
+      #   Kotoshu::Algorithms::Permutations.mapchars("anarchia", [Set.new(['a', 'á', 'ã'])]) do |variant|
+      #     puts variant
+      #   end
+      def mapchars(word, maptable)
+        return if word.length < 2 || maptable.nil? || maptable.empty?
+
+        mapchars_internal(word, 0, maptable) { |variant| yield variant }
+      end
+
+      # Produces permutations with adjacent chars swapped.
+      #
+      # For short (4 or 5 letters) words also produces double swaps: ahev -> have
+      #
+      # @param word [String] The word to process
+      # @yield [String] Each swap variant
+      def swapchar(word)
+        return if word.length < 2
+
+        chars = word.chars
+        (0...chars.length - 1).each do |i|
+          swapped = chars[0...i] + [chars[i + 1], chars[i]] + chars[(i + 2)..]
+          yield swapped.join
+        end
+
+        # Try double swaps for short words
+        # ahev -> have, owudl -> would
+        if [4, 5].include?(word.length)
+          yield word[1] + word[0] + (word.length == 5 ? word[2] : '') + word[-1] + word[-2]
+          if word.length == 5
+            yield word[0] + word[2] + word[1] + word[-1] + word[-2]
+          end
+        end
+      end
+
+      # Produces permutations with non-adjacent chars swapped (up to 4 chars distance).
+      #
+      # @param word [String] The word to process
+      # @yield [String] Each long swap variant
+      def longswapchar(word)
+        chars = word.chars
+        (0...chars.length - 2).each do |first|
+          ((first + 2)...[first + MAX_CHAR_DISTANCE, chars.length].min).each do |second|
+            swapped = chars[0...first] +
+                     [chars[second]] +
+                     chars[(first + 1)...second] +
+                     [chars[first]] +
+                     chars[(second + 1)..]
+            yield swapped.join
+          end
+        end
+      end
+
+      # Produces permutations with chars replaced by adjacent chars on keyboard layout
+      # ("vat -> cat") or downcased (if it was accidental uppercase).
+      #
+      # @param word [String] The word to process
+      # @param layout [String] Keyboard layout string (KEY from aff file)
+      # @yield [String] Each variant with replaced chars
+      def badcharkey(word, layout)
+        chars = word.chars
+        chars.each_with_index do |c, i|
+          before = word[0...i]
+          after = word[(i + 1)..]
+
+          # Try uppercasing if not already uppercase
+          unless c == c.upcase
+            yield before + c.upcase + after.to_s
+          end
+
+          next if layout.nil? || layout.empty?
+
+          # Try adjacent keys on keyboard
+          pos = layout.index(c)
+          next unless pos
+
+          while pos
+            if pos.positive? && layout[pos - 1] != '|'
+              yield before + layout[pos - 1] + after.to_s
+            end
+            if pos + 1 < layout.length && layout[pos + 1] != '|'
+              yield before + layout[pos + 1] + after.to_s
+            end
+            pos = layout.index(c, pos + 1)
+          end
+        end
+      end
+
+      # Produces permutations with one char removed in all possible positions.
+      #
+      # @param word [String] The word to process
+      # @yield [String] Each variant with one char removed
+      def extrachar(word)
+        return if word.length < 2
+
+        word.length.times do |i|
+          yield word[0...i] + word[(i + 1)..]
+        end
+      end
+
+      # Produces permutations with one char inserted in all possible positions.
+      #
+      # List of chars is taken from TRY string -- if absent, tries nothing.
+      # Chars are expected to be sorted in order of usage in language.
+      #
+      # @param word [String] The word to process
+      # @param trystring [String] Characters to try inserting (from aff TRY directive)
+      # @yield [String] Each variant with one char inserted
+      def forgotchar(word, trystring)
+        return if trystring.nil? || trystring.empty?
+
+        trystring.each_char do |c|
+          (0..word.length).each do |i|
+            yield word[0...i] + c + word[i..]
+          end
+        end
+      end
+
+      # Produces permutations with one character moved by 2, 3 or 4 places forward or backward
+      # (not 1, because adjacent swaps are already handled by swapchar).
+      #
+      # @param word [String] The word to process
+      # @yield [String] Each variant with moved character
+      def movechar(word)
+        return if word.length < 2
+
+        chars = word.chars
+
+        # Move characters forward
+        chars.each_with_index do |char, frompos|
+          ((frompos + 3)...[chars.length, frompos + MAX_CHAR_DISTANCE + 1].min).each do |topos|
+            moved = chars[0...frompos] + chars[(frompos + 1)...topos] + [char] + chars[topos..]
+            yield moved.join
+          end
+        end
+
+        # Move characters backward
+        (chars.length - 1).downto(0) do |frompos|
+          [[0, frompos - MAX_CHAR_DISTANCE + 1].max, frompos - 1].min.downto(0) do |topos|
+            moved = chars[0...topos] + [chars[frompos]] + chars[topos...frompos] + chars[(frompos + 1)..]
+            yield moved.join
+          end
+        end
+      end
+
+      # Produces permutations with chars replaced by chars in TRY set.
+      #
+      # @param word [String] The word to process
+      # @param trystring [String] Characters to try replacing with (from aff TRY directive)
+      # @yield [String] Each variant with replaced char
+      def badchar(word, trystring)
+        return if trystring.nil? || trystring.empty?
+
+        trystring.each_char do |c|
+          (word.length - 1).downto(0) do |i|
+            next if word[i] == c
+
+            yield word[0...i] + c + word[(i + 1)..]
+          end
+        end
+      end
+
+      # Produces permutations with accidental two-letter-doubling fixed.
+      # Example: "vacacation" -> "vacation"
+      #
+      # @param word [String] The word to process
+      # @yield [String] Each variant with fixed doubling
+      def doubletwochars(word)
+        return if word.length < 5
+
+        (2...word.length).each do |i|
+          # Check if word[i-2] == word[i] and word[i-3] == word[i-1]
+          # Example: vacacation -> "ca" at positions 3-4, so "vac" at 2-4
+          if word[i - 2] == word[i] && word[i - 3] == word[i - 1]
+            yield word[0...(i - 1)] + word[(i + 1)..]
+          end
+        end
+      end
+
+      # Produces permutations of splitting word into two in all possible positions.
+      #
+      # @param word [String] The word to process
+      # @yield [Array<String>] Each two-word split
+      def twowords(word)
+        (1...word.length).each do |i|
+          yield [word[0...i], word[i..]]
+        end
+      end
+
+      # Internal recursive method for mapchars.
+      #
+      # @param word [String] Current word state
+      # @param start [Integer] Starting position for search
+      # @param maptable [Array<Set<String>>] Character mapping table
+      # @yield [String] Each variant
+      def mapchars_internal(word, start, maptable)
+        return if start >= word.length
+
+        maptable.each do |options|
+          options.each do |option|
+            pos = word.index(option, start)
+            next unless pos
+
+            options.each do |other|
+              next if other == option
+
+              replaced = word[0...pos] + other + word[(pos + option.length)..]
+              yield replaced
+
+              # Recursively continue from this position
+              mapchars_internal(replaced, pos + 1, maptable) { |variant| yield variant }
+            end
+          end
+        end
+      end
+
+      private_class_method :mapchars_internal
+    end
+  end
+end