kotoshu 0.3.0

data/lib/kotoshu/readers/file_reader.rb

@@ -0,0 +1,347 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+
+begin
+  require 'zip'
+rescue LoadError
+  # rubyzip is optional — only needed for ZipReader (.oxt dictionaries).
+  # The plain FileReader and StreamReader work without it.
+end
+
+module Kotoshu
+  module Readers
+    # Base reader class for reading files line by line.
+    #
+    # This class provides:
+    # - Line-by-line reading with line numbers
+    # - BOM (byte-order mark) handling
+    # - Comment stripping
+    # - Empty line filtering
+    #
+    # @example Basic usage
+    #   reader = FileReader.new('file.aff', 'UTF-8')
+    #   reader.each do |line_no, line|
+    #     puts "#{line_no}: #{line}"
+    #   end
+    class FileReader
+      # @return [String] The file path
+      attr_reader :path
+
+      # @return [String] The encoding
+      attr_reader :encoding
+
+      # @return [Integer] Current line number
+      attr_reader :line_no
+
+      # BOM (byte-order mark) for UTF-8
+      UTF8_BOM = "\xEF\xBB\xBF".freeze
+
+      # Create a new file reader.
+      #
+      # @param path [String] Path to the file
+      # @param encoding [String] File encoding (default: 'UTF-8')
+      def initialize(path, encoding = 'UTF-8')
+        @path = path
+        @encoding = encoding
+        @line_no = 0
+        @file = nil
+        @iterator = nil
+        reset_io
+      end
+
+      # Reset encoding and reopen file.
+      #
+      # @param new_encoding [String] New encoding
+      def reset_encoding(new_encoding)
+        @encoding = new_encoding
+        @line_no = 0
+        @file&.close
+        reset_io
+      end
+
+      # Iterate over lines.
+      #
+      # @yield [Integer, String] Line number and line content
+      # @return [Enumerator] If no block given
+      def each
+        return enum_for(:each) unless block_given?
+
+        @iterator.each { |line_no, line| yield(line_no, line) }
+      end
+
+      # Get all lines as an array.
+      #
+      # @return [Array<Array<Integer, String>>] Array of [line_no, line] pairs
+      def to_a
+        @iterator.to_a
+      end
+
+      # Check if there are more lines.
+      #
+      # @return [Boolean] True if there are more lines
+      def has_next?
+        peek
+        true
+      rescue StopIteration
+      false
+      end
+
+      # Peek at next line without consuming it.
+      #
+      # @return [Array<Integer, String>] Next line number and content
+      def peek
+        @iterator.peek
+      end
+
+      # Get next line.
+      #
+      # @return [Array<Integer, String>] Line number and content
+      def next
+        @iterator.next
+      end
+
+      # Reset the reader to the beginning.
+      def reset
+        @line_no = 0
+        reset_io
+      end
+
+      # Close the file.
+      def close
+        @file&.close
+        @file = nil
+      end
+
+      private
+
+      # Reset the IO object.
+      def reset_io
+        @file = File.open(@path, "r:#{@encoding}:utf-8")
+        @iterator = read_lines.lazy
+      end
+
+      # Read lines from the file.
+      #
+      # @return [Enumerator] Enumerator of [line_no, line] pairs
+      def read_lines
+        return enum_for(:read_lines) unless block_given?
+
+        @file.each_line do |line|
+          @line_no += 1
+          line = line.strip
+
+          # Skip empty lines
+          next if line.empty?
+
+          # Handle UTF-8 BOM on first line
+          if @line_no == 1 && line.start_with?(UTF8_BOM)
+            line = line[UTF8_BOM.length..]
+            line = line.strip if line
+          end
+
+          # Skip if line is now empty after processing
+          next if line.nil? || line.empty?
+
+          yield [@line_no, line]
+        end
+      end
+    end
+
+    # String reader for reading from a string.
+    #
+    # Useful for testing or when content is already in memory.
+    class StringReader < FileReader
+      # Create a new string reader.
+      #
+      # @param content [String] The content to read
+      # @param encoding [String] Encoding (default: 'UTF-8')
+      def initialize(content, encoding = 'UTF-8')
+        @content = content
+        @lines = content.split("\n", -1)
+        @index = 0
+        super(nil, encoding)
+      end
+
+      private
+
+      def reset_io
+        @line_no = 0
+        @index = 0
+        @iterator = read_lines_iterator
+      end
+
+      def read_lines_iterator
+        Enumerator.new do |yielder|
+          while @index < @lines.length
+            @line_no += 1
+            line = @lines[@index].strip
+            @index += 1
+
+            # Skip empty lines
+            next if line.empty?
+
+            # Handle UTF-8 BOM on first line
+            if @line_no == 1 && line.start_with?(UTF8_BOM)
+              line = line[UTF8_BOM.length..]
+              line = line.strip if line
+            end
+
+            # Skip if line is now empty after processing
+            next if line.nil? || line.empty?
+
+            yielder << [@line_no, line]
+          end
+        end
+      end
+    end
+
+    # Zip reader for reading files from zip archives.
+    #
+    # This class reads files from within zip archives, such as
+    # OpenOffice/LibreOffice extensions (.odt, .oxt).
+    #
+    # @example Reading from a zip archive
+    #   zip = Zip::File.open('dictionary.oxt')
+    #   reader = ZipReader.new(zip, 'en_US.aff', 'UTF-8')
+    #   reader.each do |line_no, line|
+    #     puts "#{line_no}: #{line}"
+    #   end
+    class ZipReader
+      # @return [Zip::File] The zip file object
+      attr_reader :zipfile
+
+      # @return [String] The entry path within the zip
+      attr_reader :entry_path
+
+      # @return [String] The encoding
+      attr_reader :encoding
+
+      # @return [Integer] Current line number
+      attr_reader :line_no
+
+      # BOM (byte-order mark) for UTF-8
+      UTF8_BOM = "\xEF\xBB\xBF".freeze
+
+      # Create a new zip reader.
+      #
+      # @param zipfile [Zip::File] The zip file object
+      # @param entry_path [String] Path to the entry within the zip
+      # @param encoding [String] File encoding (default: 'UTF-8')
+      def initialize(zipfile, entry_path, encoding = 'UTF-8')
+        @zipfile = zipfile
+        @entry_path = entry_path
+        @encoding = encoding
+        @line_no = 0
+        @entry = nil
+        @iterator = nil
+        reset_io
+      end
+
+      # Reset encoding and reopen zip entry.
+      #
+      # @param new_encoding [String] New encoding
+      def reset_encoding(new_encoding)
+        @encoding = new_encoding
+        @line_no = 0
+        @entry&.close
+        reset_io
+      end
+
+      # Iterate over lines.
+      #
+      # @yield [Integer, String] Line number and line content
+      # @return [Enumerator] If no block given
+      def each
+        return enum_for(:each) unless block_given?
+
+        @iterator.each { |line_no, line| yield(line_no, line) }
+      end
+
+      # Get all lines as an array.
+      #
+      # @return [Array<Array<Integer, String>>] Array of [line_no, line] pairs
+      def to_a
+        @iterator.to_a
+      end
+
+      # Check if there are more lines.
+      #
+      # @return [Boolean] True if there are more lines
+      def has_next?
+        peek
+        true
+      rescue StopIteration
+        false
+      end
+
+      # Peek at next line without consuming it.
+      #
+      # @return [Array<Integer, String>] Next line number and content
+      def peek
+        @iterator.peek
+      end
+
+      # Get next line.
+      #
+      # @return [Array<Integer, String>] Line number and content
+      def next
+        @iterator.next
+      end
+
+      # Reset the reader to the beginning.
+      def reset
+        @line_no = 0
+        reset_io
+      end
+
+      # Close the zip entry.
+      def close
+        @entry&.close
+        @entry = nil
+      end
+
+      private
+
+      # Reset the IO object.
+      def reset_io
+        @entry = @zipfile.find_entry(@entry_path)
+        raise IOError, "Entry not found in zip: #{@entry_path}" unless @entry
+
+        # Read the entire entry content and decode it
+        content = @entry.get_input_stream.read
+        content = content.encode(@encoding, invalid: :replace, undef: :replace)
+
+        @lines = content.split("\n", -1)
+        @line_no = 0
+        @iterator = read_lines_from_zip.lazy
+      end
+
+      # Read lines from the zip entry.
+      #
+      # @return [Enumerator] Enumerator of [line_no, line] pairs
+      def read_lines_from_zip
+        return enum_for(:read_lines_from_zip) unless block_given?
+
+        @lines.each do |line|
+          @line_no += 1
+          line = line.strip
+
+          # Skip empty lines
+          next if line.empty?
+
+          # Handle UTF-8 BOM on first line
+          if @line_no == 1 && line.start_with?(UTF8_BOM)
+            line = line[UTF8_BOM.length..]
+            line = line.strip if line
+          end
+
+          # Skip if line is now empty after processing
+          next if line.nil? || line.empty?
+
+          yield [@line_no, line]
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/readers/lookup_builder.rb

@@ -0,0 +1,299 @@
+# frozen_string_literal: true
+
+require_relative '../algorithms/lookup'
+require_relative '../algorithms/capitalization'
+require_relative 'aff_reader'
+require_relative 'dic_reader'
+require_relative 'condition_checker'
+
+module Kotoshu
+  module Readers
+    # Builder for creating Lookup::Lookuper instances from Hunspell data.
+    #
+    # This class can either read from files or accept pre-read aff/dic data.
+    #
+    # @example Building a lookuper from files
+    #   builder = LookupBuilder.new('en_US.aff', 'en_US.dic')
+    #   lookuper = builder.build
+    #
+    # @example Building a lookuper from pre-read data
+    #   aff_reader = AffReader.new('en_US.aff')
+    #   aff_data = aff_reader.read
+    #   dic_reader = DicReader.new('en_US.dic')
+    #   words = dic_reader.read
+    #   builder = LookupBuilder.from_data(aff_data, words)
+    #   lookuper = builder.build
+    class LookupBuilder
+      attr_reader :aff_path, :dic_path, :encoding, :aff_data, :words, :script
+
+      # Create a new LookupBuilder from file paths.
+      #
+      # @param aff_path [String] Path to the .aff file
+      # @param dic_path [String] Path to the .dic file
+      # @param encoding [String] File encoding (default: 'UTF-8')
+      # @param script [Symbol] The script type for condition checking (default: :latin)
+      def initialize(aff_path, dic_path, encoding: 'UTF-8', script: :latin)
+        @aff_path = aff_path
+        @dic_path = dic_path
+        @encoding = encoding
+        @script = script
+        @aff_data = nil
+        @words = nil
+      end
+
+      # Create a new LookupBuilder from pre-read data.
+      #
+      # @param aff_data [Hash] Raw aff data from AffReader
+      # @param words [Array<Word>] Word entries from DicReader
+      # @return [LookupBuilder] A new builder instance
+      def self.from_data(aff_data, words)
+        builder = new(nil, nil)
+        builder.instance_variable_set(:@aff_data, aff_data)
+        builder.instance_variable_set(:@words, words)
+        builder
+      end
+
+      # Build the Lookuper instance.
+      #
+      # @return [Algorithms::Lookup::Lookuper] The lookuper instance
+      def build
+        # Read files if data not already provided
+        aff_data_to_use = @aff_data || read_aff_data
+        words_to_use = @words || read_dic_data(aff_data_to_use)
+
+        # Build the aff structure for Lookuper
+        aff = build_aff_structure(aff_data_to_use)
+
+        # Build the dic structure for Lookuper
+        dic = build_dic_structure(words_to_use)
+
+        # Create and return the Lookuper
+        Algorithms::Lookup::Lookuper.new(aff, dic)
+      end
+
+      private
+
+      # Read aff data from file.
+      #
+      # @return [Hash] Raw aff data
+      def read_aff_data
+        aff_reader = AffReader.new(@aff_path, encoding: @encoding)
+        aff_reader.read
+      end
+
+      # Read dic data from file.
+      #
+      # @param aff_data [Hash] Aff data for flag format info
+      # @return [Array<Word>] Word entries
+      def read_dic_data(aff_data)
+        dic_reader = DicReader.new(@dic_path,
+                                   encoding: @encoding,
+                                   flag_format: aff_data['FLAG'] || 'short',
+                                   flag_synonyms: aff_data['AF'] || {})
+        dic_reader.read
+      end
+
+      private
+
+      # Build the aff data structure for Lookuper.
+      #
+      # @param aff_data [Hash] Raw aff data from AffReader
+      # @return [Hash] Aff structure for Lookuper
+      def build_aff_structure(aff_data)
+        aff = {}
+
+        # Capitalization handler - default to standard Casing
+        # Could be extended to use TurkicCasing or GermanCasing based on LANG
+        aff[:casing] = Algorithms::Capitalization::Casing.new
+
+        # Build suffixes index (indexed by first character of reversed suffix)
+        suffixes_index = {}
+        aff_data['SFX'].each do |_flag, affix_list|
+          affix_list.each do |affix|
+            # For suffixes, we need to index by the first char of the REVERSED suffix
+            # because the lookup code reverses the word to check suffixes
+            reversed_suffix = affix.add.reverse
+            first_char = reversed_suffix[0] || ''
+            suffixes_index[first_char] ||= []
+            suffixes_index[first_char] << build_affix_hash(affix, script: @script || :latin)
+          end
+        end
+        aff[:suffixes_index] = suffixes_index
+
+        # Build prefixes index (indexed by first character of prefix)
+        prefixes_index = {}
+        aff_data['PFX'].each do |_flag, affix_list|
+          affix_list.each do |affix|
+            first_char = affix.add[0] || ''
+            prefixes_index[first_char] ||= []
+            prefixes_index[first_char] << build_affix_hash(affix, script: @script || :latin)
+          end
+        end
+        aff[:prefixes_index] = prefixes_index
+
+        # Single-value flags
+        aff[:COMPOUNDMIN] = aff_data['COMPOUNDMIN']
+        aff[:COMPOUNDWORDMAX] = aff_data['COMPOUNDWORDMAX']
+        aff[:COMPOUNDBEGIN] = aff_data['COMPOUNDBEGIN']
+        aff[:COMPOUNDMIDDLE] = aff_data['COMPOUNDMIDDLE']
+        aff[:COMPOUNDEND] = aff_data['COMPOUNDEND']
+        aff[:COMPOUNDFLAG] = aff_data['COMPOUNDFLAG']
+        aff[:COMPOUNDPERMITFLAG] = aff_data['COMPOUNDPERMITFLAG']
+        aff[:COMPOUNDFORBIDFLAG] = aff_data['COMPOUNDFORBIDFLAG']
+        aff[:COMPOUNDRULE] = build_compound_rules(aff_data['COMPOUNDRULE'])
+        aff[:ONLYINCOMPOUND] = aff_data['ONLYINCOMPOUND']
+        aff[:COMPLEXPREFIXES] = aff_data['COMPLEXPREFIXES']
+        aff[:FORCEUCASE] = aff_data['FORCEUCASE']
+
+        # Special flags
+        aff[:FORBIDDENWORD] = aff_data['FORBIDDENWORD']
+        aff[:NOSUGGEST] = aff_data['NOSUGGEST']
+        aff[:KEEPCASE] = aff_data['KEEPCASE']
+        aff[:NEEDAFFIX] = aff_data['NEEDAFFIX']
+        aff[:CIRCUMFIX] = aff_data['CIRCUMFIX']
+        aff[:WARN] = aff_data['WARN']
+
+        # Compound checking flags
+        aff[:CHECKCOMPOUNDCASE] = aff_data['CHECKCOMPOUNDCASE']
+        aff[:CHECKCOMPOUNDDUP] = aff_data['CHECKCOMPOUNDDUP']
+        aff[:CHECKCOMPOUNDREP] = aff_data['CHECKCOMPOUNDREP']
+        aff[:CHECKCOMPOUNDTRIPLE] = aff_data['CHECKCOMPOUNDTRIPLE']
+        aff[:CHECKCOMPOUNDPATTERN] = build_compound_patterns(aff_data['CHECKCOMPOUNDPATTERN'])
+        aff[:SIMPLIFIEDTRIPLE] = aff_data['SIMPLIFIEDTRIPLE']
+
+        # Other directives
+        aff[:IGNORE] = aff_data['IGNORE']&.chars || []
+        aff[:BREAK] = build_break_patterns(aff_data['BREAK'])
+        aff[:ICONV] = aff_data['ICONV']
+        aff[:OCONV] = aff_data['OCONV']
+        aff[:REP] = aff_data['REP'] || []
+        aff[:MAP] = aff_data['MAP'] || []
+        aff[:CHECKSHARPS] = aff_data['CHECKSHARPS']
+
+        aff
+      end
+
+      # Build the dic data structure for Lookuper.
+      #
+      # @param words [Array<Word>] List of word entries
+      # @return [Hash] Dic structure for Lookuper
+      def build_dic_structure(words)
+        # Build a hash indexed by word for fast lookup
+        word_index = Hash.new { |h, k| h[k] = [] }
+
+        words.each do |word|
+          word_index[word.stem] << {
+            stem: word.stem,
+            flags: word.flags.to_a
+          }
+        end
+
+        # Build the dic structure with homonyms callable
+        {
+          homonyms: ->(word) { word_index[word] || [] },
+          has_flag: ->(word, flag, for_all: false) {
+            entries = word_index[word] || []
+            flags_present = entries.map { |e| e[:flags] }.flatten
+            if for_all
+              flags_present.all? { |flags| flags.include?(flag) }
+            else
+              flags_present.any? { |flags| flags.include?(flag) }
+            end
+          }
+        }
+      end
+
+      # Build an affix hash for Lookuper.
+      #
+      # @param affix [Affix] The affix object
+      # @param script [Symbol] The script type for condition checking
+      # @return [Hash] Affix hash for Lookuper
+      def build_affix_hash(affix, script: :latin)
+        {
+          flag: affix.flag,
+          crossproduct: affix.crossproduct,
+          strip: affix.strip,
+          affix: affix.add,
+          condition_checker: compile_condition_matcher(affix.condition, script: script),
+          affix_data: build_affix_transform(affix.strip, affix.add, type: affix.type),
+          flags: affix.flags.to_a
+        }
+      end
+
+      # Compile a condition checker.
+      #
+      # @param condition [String] Condition string from .aff file
+      # @param script [Symbol] The script type (:latin, :arabic, etc.)
+      # @return [ConditionChecker, nil] Compiled checker or nil
+      def compile_condition_matcher(condition, script: :latin)
+        return nil if condition.nil? || condition.empty?
+
+        ConditionChecker.compile(condition, script: script)
+      end
+
+      # Build affix stripping data.
+      #
+      # Build affix transformation data.
+      #
+      # @param strip [String] Characters to strip
+      # @param add [String] Characters to add
+      # @param type [Symbol] :prefix or :suffix
+      # @return [Hash] Hash with affix data for transformation
+      def build_affix_transform(strip, add, type:)
+        return nil if strip.empty? && add.empty?
+
+        {
+          add: add,
+          strip: strip || '',
+          type: type
+        }
+      end
+
+      # Build compound rules array.
+      #
+      # @param rules [Array<CompoundRule>] List of compound rules
+      # @return [Array<Hash>] Array of compound rule hashes
+      def build_compound_rules(rules)
+        return [] if rules.nil? || rules.empty?
+
+        rules.map do |rule|
+          {
+            text: rule.text,
+            flags: rule.flags,
+            full_match: ->(flag_sets) { rule.fullmatch(flag_sets) },
+            partial_match: ->(flag_sets) { rule.flags.intersect?(flag_sets.flatten.to_set) }
+          }
+        end
+      end
+
+      # Build compound patterns array.
+      #
+      # @param patterns [Array<CompoundPattern>] List of compound patterns
+      # @return [Array<Hash>] Array of compound pattern hashes
+      def build_compound_patterns(patterns)
+        return [] if patterns.nil? || patterns.empty?
+
+        patterns.map do |pattern|
+          {
+            match: ->(left, right) { pattern.match?(left, right) }
+          }
+        end
+      end
+
+      # Build break patterns array.
+      #
+      # @param break_patterns [Array<BreakPattern>] List of break patterns
+      # @return [Array<Hash>] Array of break pattern hashes
+      def build_break_patterns(break_patterns)
+        return [] if break_patterns.nil? || break_patterns.empty?
+
+        break_patterns.map do |bp|
+          {
+            pattern: bp.pattern,
+            matcher: bp.matcher
+          }
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/readers/readers.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require_relative 'file_reader'
+require_relative 'aff_data'
+require_relative 'aff_reader'
+require_relative 'dic_reader'

data/lib/kotoshu/readers.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  # Hunspell readers module for reading dictionary and affix files.
+  module Readers
+  end
+end
+
+require_relative 'readers/readers'

data/lib/kotoshu/resource_bundle.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  ResourceBundle = Struct.new(
+    :language,
+    :dictionary,
+    :frequency,
+    :model,
+    :rules,
+    :cached,
+    :source_urls,
+    keyword_init: true
+  ) do
+    def cached?
+      cached ? true : false
+    end
+
+    def has_frequency?
+      !frequency.nil?
+    end
+
+    def has_model?
+      !model.nil?
+    end
+
+    def has_rules?
+      !rules.nil?
+    end
+  end
+end