kotoshu 0.3.0

data/lib/kotoshu/dictionary/unified.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+require 'zip'
+require_relative 'readers/file_reader'
+require_relative 'readers/aff_reader'
+require_relative 'readers/dic_reader'
+require_relative 'readers/lookup_builder'
+require_relative 'algorithms/lookup'
+require_relative 'algorithms/suggest'
+
+module Kotoshu
+  # Unified Dictionary interface matching Spylls API.
+  #
+  # This class provides the main interface to Hunspell dictionaries,
+  # supporting loading from files, zip archives, and system paths.
+  #
+  # @example Loading from files
+  #   dictionary = Dictionary.from_files('/path/to/dictionary/en_US')
+  #   dictionary.lookup('spells')  # => true
+  #
+  # @example Loading from zip archive
+  #   dictionary = Dictionary.from_zip('/path/to/dictionary/en_US.odt')
+  #
+  # @example Loading from system
+  #   dictionary = Dictionary.from_system('en_US')
+  #
+  # @example Getting suggestions
+  #   dictionary.suggest('spylls')  # => ["spells", "spills", ...]
+  #
+  # @example Accessing algorithms for experimentation
+  #   dictionary.lookuper.good_forms('building') do |form|
+  #     puts form
+  #   end
+  class Dictionary
+    # System paths to search for Hunspell dictionaries
+    PATHES = [
+      '/usr/share/hunspell',
+      '/usr/share/myspell',
+      '/usr/share/myspell/dicts',
+      '/Library/Spelling',
+      '/opt/openoffice.org/basis3.0/share/dict/ooo',
+      '/usr/lib/openoffice.org/basis3.0/share/dict/ooo',
+      '/opt/openoffice.org2.4/share/dict/ooo',
+      '/usr/lib/openoffice.org2.4/share/dict/ooo',
+      '/opt/openoffice.org2.3/share/dict/ooo',
+      '/usr/lib/openoffice.org2.3/share/dict/ooo',
+      '/opt/openoffice.org2.2/share/dict/ooo',
+      '/usr/lib/openoffice.org2.2/share/dict/ooo',
+      '/opt/openoffice.org2.1/share/dict/ooo',
+      '/usr/lib/openoffice.org2.1/share/dict/ooo',
+      '/opt/openoffice.org2.0/share/dict/ooo',
+      '/usr/lib/openoffice.org2.0/share/dict/ooo'
+    ].freeze
+
+    # Distributed dictionaries for testing
+    DISTRIBUTED = {
+      'en_US' => 'en',
+      'ru' => 'ru',
+      'sv_SE' => 'sv'
+    }.freeze
+
+    # @return [Hash] Aff data structure
+    attr_reader :aff
+
+    # @return [Array<Readers::Word>] Dic data structure
+    attr_reader :dic_words
+
+    # @return [Algorithms::Lookup::Lookuper] Lookuper instance for experimentation
+    attr_reader :lookuper
+
+    # @return [Algorithms::Suggest::Suggester] Suggester instance for experimentation
+    attr_reader :suggester
+
+    # Create a Dictionary from aff and dic data.
+    #
+    # @param aff [Hash] Aff data structure
+    # @param dic_words [Array<Readers::Word>] Dictionary word entries
+    def initialize(aff, dic_words)
+      @aff = aff
+      @dic_words = dic_words
+
+      # Create lookuper and suggester
+      @lookuper = Readers::LookupBuilder.from_data(aff, dic_words).build
+      @suggester = Algorithms::Suggest::Suggester.new(
+        aff: aff,
+        dic: build_dic_structure(dic_words),
+        lookuper: @lookuper
+      )
+    end
+
+    # Load dictionary from file path.
+    #
+    # The path should be the base name without extension, e.g., 'en_US'
+    # for files 'en_US.aff' and 'en_US.dic'.
+    #
+    # @param path [String] Base path to dictionary files (without extension)
+    # @return [Dictionary] The loaded dictionary
+    #
+    # @example
+    #   Dictionary.from_files('en_US')
+    def self.from_files(path)
+      # Check if it's a distributed dictionary
+      if DISTRIBUTED.key?(path) && !File.exist?("#{path}.aff")
+        distributed_path = File.join(File.dirname(__FILE__), '../../data', DISTRIBUTED[path], path)
+        if File.exist?("#{distributed_path}.aff")
+          path = distributed_path
+        end
+      end
+
+      aff_path = "#{path}.aff"
+      dic_path = "#{path}.dic"
+
+      raise ArgumentError, "Dictionary file not found: #{aff_path}" unless File.exist?(aff_path)
+      raise ArgumentError, "Dictionary file not found: #{dic_path}" unless File.exist?(dic_path)
+
+      # Read aff file
+      aff_reader = Readers::AffReader.new(aff_path)
+      aff_data = aff_reader.read
+
+      # Read dic file
+      dic_reader = Readers::DicReader.new(dic_path,
+                                           flag_format: aff_data['FLAG'] || 'short',
+                                           flag_synonyms: aff_data['AF'] || {})
+      dic_words = dic_reader.read
+
+      new(aff_data, dic_words)
+    end
+
+    # Load dictionary from zip archive.
+    #
+    # Supports OpenOffice/LibreOffice dictionary extensions (.odt, .oxt)
+    # and Firefox/Thunderbird dictionary extensions (.xpi).
+    #
+    # @param zip_path [String] Path to zip archive
+    # @return [Dictionary] The loaded dictionary
+    #
+    # @example
+    #   Dictionary.from_zip('en_US.odt')
+    def self.from_zip(zip_path)
+      Zip::File.open(zip_path) do |zipfile|
+        # Find .aff and .dic files
+        aff_entry = nil
+        dic_entry = nil
+
+        zipfile.each do |entry|
+          if entry.name.end_with?('.aff')
+            raise ArgumentError, "Multiple .aff files found in zip" if aff_entry
+            aff_entry = entry
+          elsif entry.name.end_with?('.dic')
+            raise ArgumentError, "Multiple .dic files found in zip" if dic_entry
+            dic_entry = entry
+          end
+        end
+
+        raise ArgumentError, "No .aff file found in zip" unless aff_entry
+        raise ArgumentError, "No .dic file found in zip" unless dic_entry
+
+        # Read aff file
+        aff_reader = Readers::ZipReader.new(zipfile, aff_entry.name)
+        aff_data = aff_reader.to_a
+        # Parse the raw data into proper aff structure
+        aff_reader = Readers::AffReader.new(zip_path) # Temporary for context
+        aff_data = Readers::AffReader.new(aff_entry.name).read
+
+        # Read dic file
+        dic_reader = Readers::DicReader.new(dic_entry.name,
+                                             flag_format: aff_data['FLAG'] || 'short',
+                                             flag_synonyms: aff_data['AF'] || {})
+        dic_words = dic_reader.read
+
+        new(aff_data, dic_words)
+      end
+    end
+
+    # Load dictionary from system paths.
+    #
+    # Searches standard system locations for Hunspell dictionaries.
+    #
+    # @param name [String] Dictionary name (e.g., 'en_US', 'ru_RU')
+    # @return [Dictionary] The loaded dictionary
+    # @raise [ArgumentError] If dictionary not found in system paths
+    #
+    # @example
+    #   Dictionary.from_system('en_US')
+    def self.from_system(name)
+      PATHES.each do |folder|
+        aff_path = File.join(folder, "#{name}.aff")
+        if File.exist?(aff_path)
+          base_path = aff_path.sub(/\.aff$/, '')
+          return from_files(base_path)
+        end
+      end
+
+      raise ArgumentError, "#{name}.aff not found in system paths: #{PATHES.inspect}"
+    end
+
+    # Check if a word is correct.
+    #
+    # @param word [String] Word to check
+    # @return [Boolean] True if the word exists in the dictionary
+    #
+    # @example
+    #   dictionary.lookup('spells')  # => true
+    #   dictionary.lookup('spylls')  # => false
+    def lookup(word)
+      @lookuper.call(word)
+    end
+
+    # Generate suggestions for a misspelled word.
+    #
+    # Returns suggestions in order of probability/similarity,
+    # with best suggestions first.
+    #
+    # @param word [String] The misspelled word
+    # @yield [String] Each suggestion
+    # @return [Enumerator] If no block given
+    #
+    # @example
+    #   dictionary.suggest('spylls')  # => ["spells", "spills", ...]
+    def suggest(word)
+      return enum_for(:suggest, word) unless block_given?
+
+      @suggester.suggestions(word) do |suggestion|
+        yield suggestion
+      end
+    end
+
+    private
+
+    # Build dic structure for suggester.
+    #
+    # @param dic_words [Array<Readers::Word>] Dictionary word entries
+    # @return [Hash] Dic structure
+    def build_dic_structure(dic_words)
+      # Build a hash indexed by word for fast lookup
+      word_index = Hash.new { |h, k| h[k] = [] }
+
+      dic_words.each do |word|
+        word_index[word.stem] << {
+          stem: word.stem,
+          flags: word.flags.to_a
+        }
+      end
+
+      # Build the dic structure with homonyms callable
+      {
+        homonyms: ->(w) { word_index[w] || [] },
+        has_flag: ->(w, flag, for_all: false) {
+          entries = word_index[w] || []
+          flags_present = entries.map { |e| e[:flags] }.flatten
+          if for_all
+            flags_present.all? { |flags| flags.include?(flag) }
+          else
+            flags_present.any? { |flags| flags.include?(flag) }
+          end
+        }
+      }
+    end
+  end
+end

data/lib/kotoshu/dictionary/unix_words.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kotoshu
+  module Dictionary
+    # Unix system dictionary backend.
+    #
+    # This dictionary reads from Unix-style system dictionary files,
+    # typically located at `/usr/share/dict/words` or symlinks to
+    # dictionaries like `web2` (Webster's Second International).
+    #
+    # @example Using system dictionary
+    #   dict = UnixWords.new("/usr/share/dict/words", language_code: "en-US")
+    #   dict.lookup?("hello")     # => true
+    #   dict.suggest("helo")      # => ["hello", "help", "held", ...]
+    #
+    # @example Auto-detecting system dictionary
+    #   dict = UnixWords.detect(language_code: "en-US")
+    class UnixWords < Base
+      # Standard system paths to check for dictionaries.
+      SYSTEM_PATHS = [
+        "/usr/share/dict/words",
+        "/usr/share/dict/web2",
+        "/usr/share/dict/american-english",
+        "/usr/share/dict/british-english",
+        "/usr/dict/words",
+        "/System/Library/Assets/com_apple_MobileAsset_DictionaryServices_dictionaryOS/Dictionary/words" # macOS
+      ].freeze
+
+      # @return [String] The path to the dictionary file
+      attr_reader :path
+
+      # @return [Boolean] Whether lookups are case-sensitive
+      attr_reader :case_sensitive
+
+      # Create a new UnixWords dictionary.
+      #
+      # @param path [String] Path to the dictionary file
+      # @param language_code [String] The language code
+      # @param locale [String, nil] The locale (optional)
+      # @param case_sensitive [Boolean] Whether lookups are case-sensitive
+      # @param metadata [Hash] Additional metadata (optional)
+      def initialize(path, language_code:, locale: nil, case_sensitive: false, metadata: {})
+        super(language_code, locale: locale, metadata: metadata)
+
+        @path = File.expand_path(path)
+        @case_sensitive = case_sensitive
+        @words = load_words(@path)
+        @word_set = build_word_set
+
+        # Register this dictionary type
+        self.class.register_type(:unix_words) unless Dictionary.registry.key?(:unix_words)
+      end
+
+      # Check if a word exists in the dictionary.
+      #
+      # @param word [String] The word to look up
+      # @return [Boolean] True if the word exists
+      def lookup(word)
+        return false if word.nil? || word.empty?
+
+        lookup_word = @case_sensitive ? word : word.downcase
+        @word_set.key?(lookup_word)
+      end
+
+      # Generate spelling suggestions.
+      #
+      # Uses edit distance to find similar words in the dictionary.
+      #
+      # @param word [String] The misspelled word
+      # @param max_suggestions [Integer] Maximum suggestions
+      # @return [Array<String>] List of suggested words
+      def suggest(word, max_suggestions: 10)
+        return [] if word.nil? || word.empty?
+
+        # For now, use simple prefix matching and edit distance
+        # This will be improved with the suggestion algorithms
+        lookup_word = @case_sensitive ? word : word.downcase
+
+        # Find words with same prefix
+        prefix_len = [lookup_word.length - 1, 3].max
+        prefix = lookup_word[0...prefix_len]
+        candidates = @words.select { |w| w.start_with?(prefix) }
+
+        # Calculate edit distances
+        candidates.map do |dict_word|
+          dist = edit_distance(lookup_word, dict_word)
+          [dict_word, dist]
+        end.select { |_, dist| dist.positive? && dist <= 2 } # Only close matches
+                  .sort_by { |_, dist| dist }
+                  .first(max_suggestions)
+                  .map(&:first)
+      end
+
+      # Add a word to the dictionary.
+      #
+      # @param word [String] The word to add
+      # @param flags [Array<String>] Flags (ignored for UnixWords)
+      # @return [Boolean] True if added
+      def add_word(word, flags: [])
+        return false if word.nil? || word.empty?
+
+        lookup_word = @case_sensitive ? word : word.downcase
+        return false if @word_set.key?(lookup_word)
+
+        @words << lookup_word
+        @word_set[lookup_word] = @words.length - 1
+
+        true
+      end
+
+      # Remove a word from the dictionary.
+      #
+      # @param word [String] The word to remove
+      # @return [Boolean] True if removed
+      def remove_word(word)
+        return false if word.nil? || word.empty?
+
+        lookup_word = @case_sensitive ? word : word.downcase
+        return false unless @word_set.key?(lookup_word)
+
+        index = @word_set.delete(lookup_word)
+        @words.delete_at(index)
+
+        true
+      end
+
+      # Get all words in the dictionary.
+      #
+      # @return [Array<String>] All words
+      def words
+        @words.dup
+      end
+
+      # Detect system dictionary path.
+      #
+      # Checks standard system paths for an existing dictionary file.
+      #
+      # @return [String, nil] The detected path or nil
+      #
+      # @example
+      #   UnixWords.detect_system_dictionary  # => "/usr/share/dict/words"
+      def self.detect_system_dictionary
+        SYSTEM_PATHS.find { |p| File.exist?(p) }
+      end
+
+      # Create a dictionary by auto-detecting system dictionary.
+      #
+      # @param language_code [String] The language code
+      # @param locale [String, nil] The locale (optional)
+      # @param case_sensitive [Boolean] Whether lookups are case-sensitive
+      # @return [UnixWords, nil] The dictionary or nil if not found
+      #
+      # @example
+      #   dict = UnixWords.detect(language_code: "en-US")
+      def self.detect(language_code:, locale: nil, case_sensitive: false)
+        path = detect_system_dictionary
+        return nil unless path
+
+        new(path, language_code: language_code, locale: locale,
+                  case_sensitive: case_sensitive)
+      end
+
+      private
+
+      # Load words from dictionary file.
+      #
+      # @param path [String] The file path
+      # @return [Array<String>] List of words
+      def load_words(path)
+        raise DictionaryNotFoundError, path unless File.exist?(path)
+
+        File.foreach(path, chomp: true)
+            .reject { |line| line.empty? || line.start_with?("#") }
+            .map { |word| @case_sensitive ? word : word.downcase }
+      end
+
+      # Build a hash set for O(1) lookups.
+      #
+      # @return [Hash] Word to index mapping
+      def build_word_set
+        @words.each_with_index.to_h
+      end
+
+      # Calculate Levenshtein edit distance.
+      #
+      # @param str1 [String] First string
+      # @param str2 [String] Second string
+      # @return [Integer] Edit distance
+      def edit_distance(str1, str2)
+        return str2.length if str1.empty?
+        return str1.length if str2.empty?
+
+        # Use smaller string for inner loop
+        str1, str2 = str2, str1 if str1.length > str2.length
+
+        previous = (0..str1.length).to_a
+
+        str2.each_char.with_index do |char2, j|
+          current = [j + 1]
+
+          str1.each_char.with_index do |char1, i|
+            insert_cost = current[i] + 1
+            delete_cost = previous[i + 1] + 1
+            substitute_cost = previous[i] + (char1 == char2 ? 0 : 1)
+
+            current << [insert_cost, delete_cost, substitute_cost].min
+          end
+
+          previous = current
+        end
+
+        previous.last
+      end
+    end
+  end
+end