kotoshu 0.3.0

data/lib/kotoshu/models/nearest_neighbor.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative 'word_embedding'
+
+module Kotoshu
+  module Models
+    # Value object for embedding search results (nearest neighbors).
+    #
+    # Represents a single suggestion from semantic similarity search,
+    # with similarity score and optional embedding reference.
+    #
+    # @example Creating a neighbor
+    #   neighbor = NearestNeighbor.new("hello", 0.85, embedding: emb)
+    #   neighbor.to_s  # => "hello [85%]"
+    class NearestNeighbor
+      attr_reader :word, :similarity, :distance, :embedding
+
+      # Create a new nearest neighbor result.
+      #
+      # @param word [String] The suggested word
+      # @param similarity [Float] Cosine similarity (0.0 to 1.0)
+      # @param embedding [WordEmbedding, nil] Optional embedding reference
+      def initialize(word, similarity, embedding: nil)
+        raise ArgumentError, "Similarity must be 0-1" unless similarity.between?(0.0, 1.0)
+
+        @word = word
+        @similarity = similarity
+        @distance = 1.0 - similarity
+        @embedding = embedding
+        freeze
+      end
+
+      # Comparison for sorting (higher similarity = better).
+      #
+      # @param other [NearestNeighbor] Another neighbor
+      # @return [Integer] Comparison result (-1, 0, 1)
+      def <=>(other)
+        return 0 unless other.is_a?(NearestNeighbor)
+
+        # Higher similarity = better rank (sort descending)
+        other.similarity <=> @similarity
+      end
+
+      # Check if this equals another neighbor.
+      #
+      # @param other [Object] Another object
+      # @return [Boolean] True if words match
+      def ==(other)
+        return false unless other.is_a?(NearestNeighbor)
+
+        @word == other.word
+      end
+      alias_method :eql?, :==
+
+      # Hash code for hash table usage.
+      #
+      # @return [Integer] Hash code
+      def hash
+        @word.hash
+      end
+
+      # String representation with percentage.
+      #
+      # @return [String] Human-readable representation
+      def to_s
+        "#{@word} [#{(@similarity * 100).to_i}%]"
+      end
+      alias_method :inspect, :to_s
+
+      # Check if this is a high-confidence suggestion.
+      #
+      # @return [Boolean] True if similarity > 0.8
+      def high_confidence?
+        @similarity > 0.8
+      end
+
+      # Get confidence level category.
+      #
+      # @return [Symbol] :high, :medium, or :low
+      def confidence_level
+        return :high if @similarity > 0.8
+        return :medium if @similarity > 0.5
+        :low
+      end
+    end
+  end
+end

data/lib/kotoshu/models/onnx_model.rb

@@ -0,0 +1,333 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Models
+    # ONNX embedding model implementation.
+    #
+    # Loads FastText models converted to ONNX format for faster inference.
+    # Uses ONNX Runtime for efficient embedding lookup.
+    #
+    # @example Loading from file
+    #   model = OnnxModel.from_file('fasttext.en.onnx')
+    #   embedding = model.embedding_for('hello')
+    #
+    # @example Loading from GitHub (via ModelCache)
+    #   model = OnnxModel.from_github('en')
+    #   neighbors = model.nearest_neighbors('hello', k: 10)
+    class OnnxModel < EmbeddingModel
+      # Soft-load onnxruntime. The gem is intentionally NOT a hard runtime
+      # dependency — it fails to build on some platforms and would block
+      # install for users who only want traditional spell-checking. Semantic
+      # features light up automatically when the gem is present.
+      #
+      # KOTOSHU_NO_ONNX=1 forces semantic analysis off even when the gem is
+      # installed (useful for benchmarks / CI determinism).
+      ONNX_LOADED = begin
+        if ENV["KOTOSHU_NO_ONNX"] == "1"
+          false
+        else
+          require "onnxruntime"
+          true
+        end
+      rescue LoadError
+        false
+      end
+
+      # Error raised when semantic features are requested but onnxruntime
+      # is unavailable. Caller-friendly message points at the fix.
+      class OnnxUnavailable < Kotoshu::Error
+        def initialize(detail = nil)
+          message = "onnxruntime gem not loaded"
+          message += " (#{detail})" if detail
+          message += ". Install with: gem install onnxruntime"
+          message += ". Or set KOTOSHU_NO_ONNX=1 to silence this in code paths that opt out."
+          super(message)
+        end
+      end
+
+      # Default dimension for FastText models
+      DEFAULT_DIMENSION = 300
+
+      attr_reader :onnx_path, :vocabulary, :embedding_matrix
+
+      # Create a new ONNX model.
+      #
+      # @param language_code [String] ISO 639-1 language code
+      # @param dimension [Integer] Vector dimension
+      # @param onnx_path [String] Path to .onnx file
+      # @param vocabulary [Hash<String, Integer>] Word-to-index mapping
+      # @param embedding_matrix [Numo::SFloat] Pre-loaded embeddings (optional)
+      def initialize(language_code:, dimension: DEFAULT_DIMENSION, onnx_path:, vocabulary:, embedding_matrix: nil)
+        super(language_code: language_code, dimension: dimension)
+        @onnx_path = onnx_path
+        @vocabulary = vocabulary.freeze
+        @vocabulary_size = @vocabulary.size
+
+        # Pre-load embedding matrix if provided (for faster nearest neighbor search)
+        @embedding_matrix = embedding_matrix
+
+        # Lazy load session
+        @session = nil
+        @loaded = false
+      end
+
+      # Load ONNX model from a file.
+      #
+      # @param onnx_path [String] Path to .onnx file
+      # @param language_code [String] Language code (auto-detected from filename)
+      # @return [OnnxModel] Loaded model
+      # @raise [ArgumentError] if file doesn't exist
+      def self.from_file(onnx_path, language_code: nil)
+        raise ArgumentError, "File not found: #{onnx_path}" unless File.exist?(onnx_path)
+
+        # Detect language from filename if not provided
+        language_code ||= detect_language_from_path(onnx_path)
+
+        # Load vocabulary from .vocab.json file
+        vocab_path = onnx_path.sub('.onnx', '.vocab.json')
+        unless File.exist?(vocab_path)
+          raise ArgumentError, "Vocabulary file not found: #{vocab_path}"
+        end
+
+        require 'json'
+        vocabulary = JSON.parse(File.read(vocab_path))
+
+        # Load metadata
+        metadata_path = onnx_path.sub('.onnx', '.metadata.json')
+        dimension = DEFAULT_DIMENSION
+
+        if File.exist?(metadata_path)
+          metadata = JSON.parse(File.read(metadata_path))
+          dimension = metadata['dimension']
+        end
+
+        new(
+          language_code: language_code,
+          dimension: dimension,
+          onnx_path: onnx_path,
+          vocabulary: vocabulary
+        )
+      end
+
+      # Load ONNX model from GitHub (via ModelCache).
+      #
+      # Downloads the .onnx file from kotoshu/dictionaries repository.
+      #
+      # @param language_code [String] ISO 639-1 language code (de, en, es, fr, pt, ru)
+      # @param cache [ModelCache, nil] Optional cache instance
+      # @return [OnnxModel] Loaded model
+      # @raise [ArgumentError] if language not supported
+      def self.from_github(language_code, cache: nil)
+        require_relative '../cache/model_cache'
+
+        cache ||= Cache::ModelCache.new
+
+        # Get the .onnx file path from cache
+        onnx_file = cache.get_onnx_model(language_code)
+
+        from_file(onnx_file, language_code: language_code)
+      end
+
+      # Get embedding vector for a word.
+      #
+      # @param word [String] The word to lookup
+      # @return [WordEmbedding, nil] Embedding vector or nil if not found
+      def embedding_for(word)
+        return nil if word.nil? || word.empty?
+
+        index = @vocabulary[word]
+        return nil unless index
+
+        # Get embedding from ONNX model
+        vector = get_embedding_vector(index)
+
+        WordEmbedding.new(word, vector, @language_code, dimension: @dimension)
+      end
+
+      # Get the vocabulary (all words in the model).
+      #
+      # @return [Array<String>] Vocabulary words
+      def vocabulary
+        @vocabulary.keys
+      end
+
+      # Check if model is loaded.
+      #
+      # @return [Boolean] True if ONNX session is loaded
+      def loaded?
+        @loaded
+      end
+
+      # Find k nearest neighbors for a word.
+      #
+      # @param word [String] The query word
+      # @param k [Integer] Number of neighbors to return
+      # @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
+      def nearest_neighbors(word, k: 10)
+        ensure_session_loaded
+
+        # Get query embedding
+        query = embedding_for(word)
+        return [] unless query
+
+        # If embedding matrix is pre-loaded, use it for faster search
+        if @embedding_matrix
+          nearest_neighbors_from_matrix(query, k)
+        else
+          super
+        end
+      end
+
+      # Batch lookup of embeddings for multiple words.
+      #
+      # More efficient than individual lookups when using ONNX.
+      #
+      # @param words [Array<String>] Words to lookup
+      # @return [Hash<String, WordEmbedding>] Word to embedding mapping
+      def batch_embeddings(words)
+        ensure_session_loaded
+
+        indices = words.map { |w| @vocabulary[w] }
+        vectors = batch_get_embeddings(indices)
+
+        words.zip(indices, vectors).each_with_object({}) do |(word, idx, vec)|
+          next unless idx && vec
+
+          [word, WordEmbedding.new(word, vec, @language_code, dimension: @dimension)]
+        end
+      end
+
+      # Preload the embedding matrix into memory for faster nearest neighbor search.
+      #
+      # Useful when doing many nearest neighbor queries.
+      #
+      # @return [Boolean] True if loaded successfully
+      def preload_embedding_matrix
+        ensure_session_loaded
+
+        # Get all embeddings at once
+        all_indices = (0...@vocabulary_size).to_a
+        vectors = batch_get_embeddings(all_indices)
+
+        # Convert to matrix (using Numo::SFloat for efficiency)
+        require 'numo/narray'
+        @embedding_matrix = Numo::Sfloat.cast(vectors).reshape(@vocabulary_size, @dimension)
+
+        true
+      rescue StandardError => e
+        warn "Failed to preload embedding matrix: #{e.message}"
+        false
+      end
+
+      private
+
+      # Get embedding vector from ONNX model.
+      #
+      # @param index [Integer] Word index
+      # @return [Array<Float>] Embedding vector
+      def get_embedding_vector(index)
+        ensure_session_loaded
+
+        result = @session.run(
+          ['embeddings'],
+          { word_indices: [index].pack('q<') }  # Pack int64 as little-endian
+        )
+
+        # Unpack float32 array
+        result.first.unpack('e*')
+      end
+
+      # Get embeddings for multiple indices.
+      #
+      # @param indices [Array<Integer>] Word indices
+      # @return [Array<Array<Float>>] Embedding vectors
+      def batch_get_embeddings(indices)
+        ensure_session_loaded
+
+        valid_indices = indices.compact
+
+        return [] if valid_indices.empty?
+
+        # Pack indices as int64 array
+        input_data = valid_indices.pack('q<*')
+
+        result = @session.run(
+          ['embeddings'],
+          { word_indices: input_data }
+        )
+
+        # Unpack float32 matrix
+        vectors = result.first.unpack('e*')
+        chunk_size = @dimension
+
+        vectors.each_slice(chunk_size).to_a
+      end
+
+      # Find nearest neighbors using pre-loaded embedding matrix.
+      #
+      # @param query [WordEmbedding] Query embedding
+      # @param k [Integer] Number of neighbors
+      # @return [Array<NearestNeighbor>] Nearest neighbors
+      def nearest_neighbors_from_matrix(query, k)
+        return [] unless @embedding_matrix
+
+        # Compute cosine similarity with all words
+        query_vec = Numo::Sfloat.cast(query.vector)
+        similarities = []
+
+        @vocabulary.each_with_index do |(word, idx)|
+          vec = @embedding_matrix[idx, true]
+          sim = cosine_similarity(query_vec, vec)
+          similarities << [word, sim]
+        end
+
+        # Sort by similarity and take top k
+        similarities.sort_by { |_, s| -s }.first(k).map do |word, sim|
+          NearestNeighbor.new(
+            word: word,
+            similarity: sim,
+            embedding: embedding_for(word)
+          )
+        end
+      end
+
+      # Calculate cosine similarity between two vectors.
+      #
+      # @param vec1 [Numo::SFloat] First vector
+      # @param vec2 [Numo::SFloat] Second vector
+      # @return [Float] Cosine similarity
+      def cosine_similarity(vec1, vec2)
+        dot = (vec1 * vec2).sum
+        norm1 = Math.sqrt((vec1 ** 2).sum)
+        norm2 = Math.sqrt((vec2 ** 2).sum)
+
+        return 0.0 if norm1.zero? || norm2.zero?
+
+        dot / (norm1 * norm2)
+      end
+
+      # Ensure ONNX session is loaded.
+      def ensure_session_loaded
+        return if @loaded
+
+        raise OnnxUnavailable unless ONNX_LOADED
+
+        @session = OnnxRuntime::Session.new(@onnx_path)
+        @loaded = true
+      end
+
+      # Detect language code from file path.
+      #
+      # @param path [String] File path
+      # @return [String] Detected language code
+      def self.detect_language_from_path(path)
+        # Extract from path like "fasttext.en.onnx"
+        if path =~ /\.([a-z]{2})\./i
+          Regexp.last_match(1).downcase
+        else
+          'en'  # Default to English
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/models/semantic_error.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require_relative 'context'
+require_relative 'suggestion'
+
+module Kotoshu
+  module Models
+    # Unified semantic error (NO artificial spelling/grammar split!).
+    #
+    # Represents ANY kind of language error detected through semantic analysis.
+    # Uses semantic categories instead of traditional "spelling" vs "grammar" labels.
+    #
+    # Error types (semantic categories):
+    # - :word_choice - Wrong word for context (e.g., "desert" vs "dessert")
+    # - :verb_agreement - Subject-verb mismatch (e.g., "they is" → "they are")
+    # - :tense - Temporal inconsistency (e.g., "Yesterday I will go")
+    # - :orthographic - Actual typo/misspelling (e.g., "wrold" → "world")
+    # - :preposition - Wrong preposition (e.g., "bored of" → "bored with")
+    # - :article - Wrong article (e.g., "a apple" → "an apple")
+    # - :morphology - Wrong word form (e.g., "goed" → "went")
+    # - :capitalization - Capitalization error (e.g., "i am" → "I am")
+    # - :punctuation - Punctuation error (e.g., "its" vs "it's")
+    # - :style - Style/usage suggestion
+    #
+    # @example Creating a semantic error
+    #   error = SemanticError.new(
+    #     id: "error_1",
+    #     location: Location.new(line: 5, column: 12),
+    #     original: "desert",
+    #     suggestions: [Suggestion.new("dessert", confidence: 0.92)],
+    #     error_type: :word_choice,
+    #     confidence: 0.92,
+    #     context: context
+    #   )
+    class SemanticError
+      # Error type definitions with display names
+      ERROR_TYPES = {
+        word_choice: 'Word Choice',
+        verb_agreement: 'Verb Agreement',
+        tense: 'Tense',
+        orthographic: 'Spelling',
+        preposition: 'Preposition',
+        article: 'Article',
+        morphology: 'Word Form',
+        capitalization: 'Capitalization',
+        punctuation: 'Punctuation',
+        style: 'Style'
+      }.freeze
+
+      attr_reader :id, :location, :original, :suggestions, :error_type, :confidence, :context
+
+      # Create a new semantic error.
+      #
+      # @param id [String, Symbol] Unique identifier for this error
+      # @param location [Documents::Location] Location of error in document
+      # @param original [String] The original (incorrect) word/text
+      # @param suggestions [Array<Suggestion>] Suggested corrections
+      # @param error_type [Symbol] Error type (must be in ERROR_TYPES)
+      # @param confidence [Float] Confidence score (0.0 to 1.0)
+      # @param context [Context] Context around the error
+      # @raise [ArgumentError] if error_type is invalid
+      def initialize(id:, location:, original:, suggestions:, error_type:, confidence:, context:)
+        raise ArgumentError, "Invalid error type: #{error_type}" unless ERROR_TYPES.key?(error_type)
+        raise ArgumentError, "Confidence must be 0-1" unless confidence.between?(0.0, 1.0)
+        raise ArgumentError, "Suggestions cannot be empty" if suggestions.nil? || suggestions.empty?
+
+        @id = id.to_s
+        @location = location
+        @original = original
+        @suggestions = suggestions.sort_by(&:confidence).reverse.freeze
+        @error_type = error_type
+        @confidence = confidence
+        @context = context
+
+        freeze
+      end
+
+      # Get user-friendly display type name.
+      #
+      # @return [String] Display type name
+      def display_type
+        ERROR_TYPES[@error_type] || @error_type.to_s.capitalize
+      end
+
+      # Check if this is a high-confidence error.
+      #
+      # @return [Boolean] True if confidence > 0.8
+      def high_confidence?
+        @confidence > 0.8
+      end
+
+      # Get confidence level category.
+      #
+      # @return [Symbol] :high, :medium, or :low
+      def confidence_level
+        return :high if @confidence > 0.8
+        return :medium if @confidence > 0.5
+        :low
+      end
+
+      # Get the recommended (top) suggestion.
+      #
+      # @return [Suggestion] The highest-confidence suggestion
+      def recommended_suggestion
+        @suggestions.first
+      end
+
+      # Check if this error equals another.
+      #
+      # @param other [Object] Another object
+      # @return [Boolean] True if IDs match
+      def ==(other)
+        return false unless other.is_a?(SemanticError)
+
+        @id == other.id
+      end
+      alias_method :eql?, :==
+
+      # Hash code for hash table usage.
+      #
+      # @return [Integer] Hash code
+      def hash
+        @id.hash
+      end
+
+      # Comparison for sorting (by location, then confidence).
+      #
+      # Errors are sorted by:
+      # 1. Document location (line number, then column)
+      # 2. Confidence (highest first)
+      #
+      # @param other [SemanticError] Another error
+      # @return [Integer] Comparison result (-1, 0, 1)
+      def <=>(other)
+        return 0 unless other.is_a?(SemanticError)
+
+        # First by location (line, then column)
+        loc_cmp = @location <=> other.location
+        return loc_cmp unless loc_cmp.zero?
+
+        # Then by confidence (highest first)
+        other.confidence <=> @confidence
+      end
+
+      # String representation.
+      #
+      # @return [String] Human-readable representation
+      def to_s
+        "#{@location}: '#{@original}' → #{recommended_suggestion.word} [#{(@confidence * 100).to_i}%]"
+      end
+      alias_method :inspect, :to_s
+
+      # Create an abbreviated display for lists.
+      #
+      # @param max_length [Integer] Maximum line length
+      # @return [String] Abbreviated representation
+      def abbreviated(max_length: 80)
+        orig_display = "'#{@original}'"
+        sugg_display = "'#{recommended_suggestion.word}'"
+
+        "#{@location}: #{orig_display} → #{sugg_display} [#{(@confidence * 100).to_i}%]"
+      end
+    end
+  end
+end

data/lib/kotoshu/models/suggestion.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Models
+    # Value object for correction suggestions.
+    #
+    # Represents a suggested correction for a detected error,
+    # with confidence score and metadata.
+    #
+    # @example Creating a suggestion
+    #   suggestion = Suggestion.new("dessert", confidence: 0.92, source: :semantic)
+    #   suggestion.to_s  # => "dessert [92%]"
+    class Suggestion
+      attr_reader :word, :confidence, :source, :metadata
+
+      # Create a new suggestion.
+      #
+      # @param word [String] The suggested word
+      # @param confidence [Float] Confidence score (0.0 to 1.0)
+      # @param source [Symbol, nil] Source of the suggestion (e.g., :semantic, :edit_distance)
+      # @param metadata [Hash] Additional metadata (optional)
+      # @option metadata [WordEmbedding, nil] :embedding The word embedding
+      # @option metadata [Float] :edit_distance Edit distance score
+      # @option metadata [Float] :frequency_bonus Frequency score bonus
+      # @option metadata [String] :explanation Explanation for the suggestion
+      def initialize(word, confidence:, source: nil, metadata: {})
+        raise ArgumentError, "Confidence must be 0-1" unless confidence.between?(0.0, 1.0)
+
+        @word = word
+        @confidence = confidence
+        @source = source || :unknown
+        @metadata = metadata.freeze
+        freeze
+      end
+
+      # Comparison for sorting (higher confidence = better).
+      #
+      # @param other [Suggestion] Another suggestion
+      # @return [Integer] Comparison result (-1, 0, 1)
+      def <=>(other)
+        return 0 unless other.is_a?(Suggestion)
+
+        # Higher confidence = better rank (sort descending)
+        other.confidence <=> @confidence
+      end
+
+      # Check if this equals another suggestion.
+      #
+      # @param other [Object] Another object
+      # @return [Boolean] True if words match
+      def ==(other)
+        return false unless other.is_a?(Suggestion)
+
+        @word == other.word
+      end
+      alias_method :eql?, :==
+
+      # Hash code for hash table usage.
+      #
+      # @return [Integer] Hash code
+      def hash
+        @word.hash
+      end
+
+      # String representation with percentage.
+      #
+      # @return [String] Human-readable representation
+      def to_s
+        if @source && @source != :unknown
+          "#{@word} [#{(@confidence * 100).to_i}%] (#{@source})"
+        else
+          "#{@word} [#{(@confidence * 100).to_i}%]"
+        end
+      end
+      alias_method :inspect, :to_s
+
+      # Get the embedding if available.
+      #
+      # @return [WordEmbedding, nil] The embedding or nil
+      def embedding
+        @metadata[:embedding]
+      end
+
+      # Get the edit distance if available.
+      #
+      # @return [Float, nil] Edit distance or nil
+      def edit_distance
+        @metadata[:edit_distance]
+      end
+
+      # Check if this is a high-confidence suggestion.
+      #
+      # @return [Boolean] True if confidence > 0.8
+      def high_confidence?
+        @confidence > 0.8
+      end
+
+      # Get explanation text if available.
+      #
+      # @return [String, nil] Explanation or nil
+      def explanation
+        @metadata[:explanation]
+      end
+    end
+  end
+end