kotoshu 0.3.0

data/lib/kotoshu/metrics_module.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  # Metrics and instrumentation for Kotoshu.
+  #
+  # Provides thread-safe collection of performance metrics:
+  # - Lookup counts and timing
+  # - Cache hit/miss rates
+  # - Suggestion generation stats
+  # - Optional export to StatsD or Prometheus
+  #
+  # @example Enable metrics
+  #   Kotoshu::Metrics.enable
+  #   Kotoshu.correct?("hello")
+  #   Kotoshu::Metrics.stats
+  #   # => { lookups: 1, cache_hits: 0, cache_misses: 1, ... }
+  module Metrics
+    class << self
+      # Enable metrics collection.
+      def enable
+        @enabled = true
+        @collector = Collector.new
+      end
+
+      # Disable metrics collection.
+      def disable
+        @enabled = false
+        @collector = nil
+      end
+
+      # Check if metrics are enabled.
+      #
+      # @return [Boolean] True if enabled
+      def enabled?
+        @enabled ||= false
+      end
+
+      # Get the metrics collector.
+      #
+      # @return [Collector, nil] The collector instance
+      attr_reader :collector
+
+      # Record a lookup operation.
+      #
+      # @param word [String] The word being looked up
+      # @param result [Boolean] The lookup result
+      # @param time [Float] Time taken in milliseconds
+      def record_lookup(word, result:, time:)
+        return unless enabled?
+
+        collector&.record_lookup(word, result: result, time: time)
+      end
+
+      # Record a cache operation.
+      #
+      # @param cache_type [String] Type of cache (lookup, suggestion)
+      # @param hit [Boolean] True if cache hit
+      def record_cache(cache_type, hit:)
+        return unless enabled?
+
+        collector&.record_cache(cache_type, hit: hit)
+      end
+
+      # Record suggestion generation.
+      #
+      # @param word [String] The input word
+      # @param count [Integer] Number of suggestions generated
+      # @param time [Float] Time taken in milliseconds
+      def record_suggestions(word, count:, time:)
+        return unless enabled?
+
+        collector&.record_suggestions(word, count: count, time: time)
+      end
+
+      # Get current metrics statistics.
+      #
+      # @return [Hash] Current statistics
+      def stats
+        return {} unless enabled?
+
+        collector&.stats || {}
+      end
+
+      # Reset all metrics.
+      def reset
+        return unless enabled?
+
+        collector&.reset
+      end
+
+      # Get metrics as StatsD format.
+      #
+      # @return [String] StatsD protocol lines
+      def to_statsd
+        return "" unless enabled?
+
+        collector&.to_statsd || ""
+      end
+
+      # Get metrics as Prometheus format.
+      #
+      # @return [String] Prometheus exposition format
+      def to_prometheus
+        return "" unless enabled?
+
+        collector&.to_prometheus || ""
+      end
+    end
+  end
+end

data/lib/kotoshu/models/context.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Models
+    # Value object for text context around an error.
+    #
+    # Provides the surrounding text before, current, and after
+    # an error location for context display and analysis.
+    #
+    # @example Creating context
+    #   context = Context.new(
+    #     before: "The quick brown",
+    #     current: "fox",
+    #     after: "jumps over",
+    #     location: Location.new(line: 5, column: 16)
+    #   )
+    #   context.full_context  # => "The quick brown fox jumps over"
+    class Context
+      attr_reader :before, :current, :after, :location, :window, :full_context
+
+      # Create a new context object.
+      #
+      # @param before [String] Text before the error
+      # @param current [String] The current line/text containing the error
+      # @param after [String] Text after the error
+      # @param location [Documents::Location] The error location
+      # @param window [Integer] Window size used for context (default: 5)
+      def initialize(before:, current:, after:, location:, window: 5)
+        @before = before
+        @current = current
+        @after = after
+        @location = location
+        @window = window
+        @full_context = [before, current, after].compact.join("\n")
+        freeze
+      end
+
+      # Get surrounding words around the error location.
+      #
+      # @param n [Integer] Number of words on each side (default: 3)
+      # @return [Array<String>] Surrounding words
+      def surrounding_words(n = 3)
+        return [] if @current.nil? || @current.empty?
+
+        words = @current.split
+        return [] if words.empty?
+
+        # Try to find the word at the error location
+        target_word = word_at_location
+        return words unless target_word
+
+        idx = words.index(target_word)
+        return words unless idx
+
+        # Get n words before and after
+        start_idx = [0, idx - n].max
+        end_idx = [words.size - 1, idx + n].min
+
+        words[start_idx..end_idx].to_a
+      end
+
+      # Get the word at the error location.
+      #
+      # @return [String, nil] The word at the error location
+      def word_at_location
+        return nil unless @location
+
+        if @location.column
+          # Get character at column
+          return @current[@location.column] if @current && @location.column < @current.length
+        end
+
+        # For node-based locations, return the current text
+        @current
+      end
+
+      # Check if this context equals another.
+      #
+      # @param other [Object] Another object
+      # @return [Boolean] True if contexts match
+      def ==(other)
+        return false unless other.is_a?(Context)
+
+        @location == other.location && @full_context == other.full_context
+      end
+      alias_method :eql?, :==
+
+      # Hash code for hash table usage.
+      #
+      # @return [Integer] Hash code
+      def hash
+        [@location, @full_context].hash
+      end
+
+      # String representation.
+      #
+      # @return [String] Human-readable representation
+      def to_s
+        if @location.line
+          "Line #{@location.line}: #{@full_context}"
+        else
+          @full_context
+        end
+      end
+      alias_method :inspect, :to_s
+
+      # Get context as a formatted string with error highlighting.
+      #
+      # @param error_word [String] The error word to highlight
+      # @return [String] Formatted context with ANSI codes
+      def with_highlight(error_word)
+        return @full_context unless error_word
+
+        # Find and highlight the error word
+        @full_context.gsub(error_word) { |m| "\033[4m#{m}\033[0m" }
+      end
+    end
+  end
+end

data/lib/kotoshu/models/embedding_model.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Models
+    # Abstract base class for word embedding models.
+    #
+    # Provides a unified interface for loading and querying word embeddings
+    # from different sources (FastText, Word2Vec, GloVe, ONNX, etc.).
+    #
+    # @example Using an embedding model
+    #   model = FastTextModel.new('cc.en.300.vec')
+    #   embedding = model.embedding_for('hello')
+    #   similarity = model.similarity('hello', 'world')
+    #   neighbors = model.nearest_neighbors('hello', k: 10)
+    #
+    # @abstract Subclasses must implement {#embedding_for} and {#vocabulary}
+    class EmbeddingModel
+      attr_reader :language_code, :dimension, :vocabulary_size
+
+      # Create a new embedding model.
+      #
+      # @param language_code [String] ISO 639-1 language code
+      # @param dimension [Integer] Vector dimensionality (e.g., 300)
+      def initialize(language_code:, dimension:)
+        raise ArgumentError, "Language code cannot be nil" if language_code.nil?
+        raise ArgumentError, "Dimension must be positive" unless dimension&.positive?
+
+        @language_code = language_code
+        @dimension = dimension
+        @vocabulary_size = 0
+        freeze
+      end
+
+      # Get embedding vector for a word.
+      #
+      # @param word [String] The word to lookup
+      # @return [WordEmbedding, nil] Embedding vector or nil if not found
+      # @abstract Subclass must implement
+      def embedding_for(word)
+        raise NotImplementedError, "#{self.class} must implement #embedding_for"
+      end
+
+      # Check if a word is in the vocabulary.
+      #
+      # @param word [String] The word to check
+      # @return [Boolean] True if word exists in vocabulary
+      def has_word?(word)
+        vocabulary.include?(word)
+      end
+
+      # Calculate cosine similarity between two words.
+      #
+      # @param word1 [String] First word
+      # @param word2 [String] Second word
+      # @return [Float, nil] Similarity score (0.0 to 1.0) or nil if words not found
+      def similarity(word1, word2)
+        emb1 = embedding_for(word1)
+        emb2 = embedding_for(word2)
+
+        return nil unless emb1 && emb2
+
+        emb1.similarity(emb2)
+      end
+
+      # Calculate Euclidean distance between two words.
+      #
+      # @param word1 [String] First word
+      # @param word2 [String] Second word
+      # @return [Float, nil] Distance or nil if words not found
+      def distance(word1, word2)
+        emb1 = embedding_for(word1)
+        emb2 = embedding_for(word2)
+
+        return nil unless emb1 && emb2
+
+        emb1.distance(emb2)
+      end
+
+      # Find k nearest neighbors for a word.
+      #
+      # @param word [String] The query word
+      # @param k [Integer] Number of neighbors to return
+      # @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
+      def nearest_neighbors(word, k: 10)
+        embedding = embedding_for(word)
+        return [] unless embedding
+
+        # Calculate similarity with all words in vocabulary
+        neighbors = vocabulary.map do |vocab_word|
+          next if vocab_word == word
+
+          vocab_embedding = embedding_for(vocab_word)
+          next unless vocab_embedding
+
+          sim = embedding.similarity(vocab_embedding)
+          NearestNeighbor.new(
+            word: vocab_word,
+            similarity: sim,
+            distance: embedding.distance(vocab_embedding),
+            embedding: vocab_embedding
+          )
+        end.compact
+
+        # Sort by similarity (descending) and take top k
+        neighbors.sort.reverse.first(k)
+      end
+
+      # Find k nearest neighbors for an embedding vector.
+      #
+      # @param embedding [WordEmbedding] The query embedding
+      # @param k [Integer] Number of neighbors to return
+      # @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
+      def nearest_neighbors_for_embedding(embedding, k: 10)
+        return [] unless embedding
+
+        # Calculate similarity with all words in vocabulary
+        neighbors = vocabulary.map do |vocab_word|
+          vocab_embedding = embedding_for(vocab_word)
+          next unless vocab_embedding
+
+          sim = embedding.similarity(vocab_embedding)
+          NearestNeighbor.new(
+            word: vocab_word,
+            similarity: sim,
+            distance: embedding.distance(vocab_embedding),
+            embedding: vocab_embedding
+          )
+        end.compact
+
+        # Sort by similarity (descending) and take top k
+        neighbors.sort.reverse.first(k)
+      end
+
+      # Get model metadata.
+      #
+      # @return [Hash] Model metadata
+      def metadata
+        {
+          language_code: @language_code,
+          dimension: @dimension,
+          vocabulary_size: @vocabulary_size,
+          model_type: self.class.name
+        }
+      end
+
+      # Get the vocabulary (all words in the model).
+      #
+      # @return [Array<String>] Vocabulary words
+      # @abstract Subclass must implement
+      def vocabulary
+        raise NotImplementedError, "#{self.class} must implement #vocabulary"
+      end
+
+      # Check if model is loaded.
+      #
+      # @return [Boolean] True if model is loaded and ready
+      def loaded?
+        @vocabulary_size&.positive? || vocabulary&.any?
+      end
+
+      # Get model statistics.
+      #
+      # @return [Hash] Statistics about the model
+      def statistics
+        {
+          language: @language_code,
+          dimension: @dimension,
+          vocabulary_size: @vocabulary_size,
+          loaded: loaded?
+        }
+      end
+
+      # String representation.
+      #
+      # @return [String] Human-readable representation
+      def to_s
+        "#{self.class.name}(language: #{@language_code}, dim: #{@dimension}, vocab: #{@vocabulary_size})"
+      end
+      alias_method :inspect, :to_s
+    end
+  end
+end

data/lib/kotoshu/models/fasttext_model.rb

@@ -0,0 +1,220 @@
+# frozen_string_literal: true
+
+require_relative 'embedding_model'
+require_relative 'word_embedding'
+require_relative 'nearest_neighbor'
+
+module Kotoshu
+  module Models
+    # FastText embedding model implementation.
+    #
+    # Loads FastText pre-trained word vectors from .vec files.
+    # Supports Common Crawl and Wikipedia trained vectors.
+    #
+    # @example Loading from file
+    #   model = FastTextModel.from_file('cc.en.300.vec')
+    #   model.embedding_for('hello')
+    #
+    # @example Loading from GitHub
+    #   model = FastTextModel.from_github('en')
+    #   model.nearest_neighbors('hello', k: 10)
+    #
+    # @see https://fasttext.cc/docs/en/crawl-vectors.html FastText crawl vectors
+    # @see https://fasttext.cc/docs/en/english-vectors.html FastText English vectors
+    class FastTextModel < EmbeddingModel
+      # Standard FastText dimension for crawl vectors
+      DEFAULT_DIMENSION = 300
+
+      # Number of vectors to load when reading from file
+      # FastText .vec files contain up to 2M words; we load a subset by default
+      DEFAULT_MAX_VECTORS = 1_000_000
+
+      attr_reader :embeddings, :max_vectors
+
+      # Create a new FastText model.
+      #
+      # @param language_code [String] ISO 639-1 language code
+      # @param dimension [Integer] Vector dimension (default: 300)
+      # @param embeddings [Hash<String, WordEmbedding>] Pre-loaded embeddings
+      # @param max_vectors [Integer] Maximum vectors to load from file
+      def initialize(language_code:, dimension: DEFAULT_DIMENSION, embeddings: {}, max_vectors: DEFAULT_MAX_VECTORS)
+        super(language_code: language_code, dimension: dimension)
+        @embeddings = embeddings.freeze
+        @max_vectors = max_vectors
+        @vocabulary_size = @embeddings.size
+      end
+
+      # Load FastText model from a .vec file.
+      #
+      # @param file_path [String] Path to FastText .vec file
+      # @param max_vectors [Integer] Maximum vectors to load (default: 1M)
+      # @param language_code [String] Language code (auto-detected from filename)
+      # @return [FastTextModel] Loaded model
+      # @raise [ArgumentError] if file doesn't exist
+      def self.from_file(file_path, max_vectors: DEFAULT_MAX_VECTORS, language_code: nil)
+        raise ArgumentError, "File not found: #{file_path}" unless File.exist?(file_path)
+
+        # Detect language from filename if not provided
+        language_code ||= detect_language_from_path(file_path)
+
+        # Parse the .vec file
+        embeddings = {}
+        dimension = nil
+        count = 0
+
+        File.open(file_path, 'r', encoding: 'UTF-8') do |file|
+          # First line: vocab_size dimension
+          first_line = file.getline
+          metadata = first_line.split
+          _vocab_size = metadata[0].to_i
+          dimension = metadata[1].to_i
+
+          # Read vectors
+          file.each_line do |line|
+            break if count >= max_vectors
+
+            parts = line.split
+            word = parts[0]
+            vector = parts[1..-1].map(&:to_f)
+
+            next unless vector.size == dimension
+
+            embeddings[word] = WordEmbedding.new(word, vector, language_code, dimension: dimension)
+            count += 1
+          end
+        end
+
+        new(language_code: language_code, dimension: dimension, embeddings: embeddings, max_vectors: max_vectors)
+      end
+
+      # Load FastText model from GitHub (via ModelCache).
+      #
+      # Downloads the .vec file from kotoshu/dictionaries repository.
+      #
+      # @param language_code [String] ISO 639-1 language code (de, en, es, fr, pt, ru)
+      # @param max_vectors [Integer] Maximum vectors to load (default: 500K for GitHub)
+      # @param cache [ModelCache, nil] Optional cache instance
+      # @return [FastTextModel] Loaded model
+      # @raise [ArgumentError] if language not supported
+      def self.from_github(language_code, max_vectors: 500_000, cache: nil)
+        require_relative '../cache/model_cache'
+
+        cache ||= Cache::ModelCache.new
+
+        # Get the .vec file path from cache
+        vec_file = cache.get_fasttext_model(language_code)
+
+        from_file(vec_file, max_vectors: max_vectors, language_code: language_code)
+      end
+
+      # Get embedding vector for a word.
+      #
+      # @param word [String] The word to lookup
+      # @return [WordEmbedding, nil] Embedding vector or nil if not found
+      def embedding_for(word)
+        return nil if word.nil? || word.empty?
+
+        # Direct lookup
+        @embeddings[word]
+      end
+
+      # Get the vocabulary (all words in the model).
+      #
+      # @return [Array<String>] Vocabulary words
+      def vocabulary
+        @embeddings.keys
+      end
+
+      # Check if model is loaded.
+      #
+      # @return [Boolean] True if embeddings are loaded
+      def loaded?
+        @embeddings&.any?
+      end
+
+      # Find k nearest neighbors for a word (optimized version).
+      #
+      # Overrides the base implementation for better performance using
+      # pre-loaded embeddings instead of repeated lookups.
+      #
+      # @param word [String] The query word
+      # @param k [Integer] Number of neighbors to return
+      # @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
+      def nearest_neighbors(word, k: 10)
+        embedding = embedding_for(word)
+        return [] unless embedding
+
+        # Calculate similarity with all words in vocabulary
+        neighbors = @embeddings.map do |vocab_word, vocab_embedding|
+          next if vocab_word == word
+
+          sim = embedding.similarity(vocab_embedding)
+          NearestNeighbor.new(
+            word: vocab_word,
+            similarity: sim,
+            embedding: vocab_embedding
+          )
+        end.compact
+
+        # Sort by similarity (descending) and take top k
+        neighbors.sort.reverse.first(k)
+      end
+
+      # Find k nearest neighbors for an embedding vector (optimized version).
+      #
+      # @param embedding [WordEmbedding] The query embedding
+      # @param k [Integer] Number of neighbors to return
+      # @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
+      def nearest_neighbors_for_embedding(embedding, k: 10)
+        return [] unless embedding
+
+        # Calculate similarity with all words in vocabulary
+        neighbors = @embeddings.map do |vocab_word, vocab_embedding|
+          sim = embedding.similarity(vocab_embedding)
+          NearestNeighbor.new(
+            word: vocab_word,
+            similarity: sim,
+            embedding: vocab_embedding
+          )
+        end.compact
+
+        # Sort by similarity (descending) and take top k
+        neighbors.sort.reverse.first(k)
+      end
+
+      # Get batch embeddings for multiple words.
+      #
+      # @param words [Array<String>] Words to lookup
+      # @return [Hash<String, WordEmbedding>] Mapping of word to embedding
+      def batch_embeddings(words)
+        words.each_with_object({}) do |word, hash|
+          emb = embedding_for(word)
+          hash[word] = emb if emb
+        end
+      end
+
+      # Get batch similarities for word pairs.
+      #
+      # @param pairs [Array<Array<String, String>>] Word pairs
+      # @return [Array<Float>] Similarity scores
+      def batch_similarities(pairs)
+        pairs.map { |word1, word2| similarity(word1, word2) }
+      end
+
+      private
+
+      # Detect language code from file path.
+      #
+      # @param path [String] File path
+      # @return [String] Detected language code
+      def self.detect_language_from_path(path)
+        # Extract from path like "cc.en.300.vec" or "wiki.de.vec"
+        if path =~ /\.([a-z]{2})\./i
+          Regexp.last_match(1).downcase
+        else
+          'en'  # Default to English
+        end
+      end
+    end
+  end
+end