kotoshu 0.3.0

data/lib/kotoshu/components/pos_tagger.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Components
+    # Base class for POS (Part-of-Speech) taggers.
+    #
+    # POS taggers assign grammatical categories (NOUN, VERB, ADJ, etc.) to tokens.
+    # Different languages use different POS tagging strategies:
+    # - Latin scripts: Dictionary-based (Hunspell flags → POS tags)
+    # - CJK: Integrated with morphological analysis (tokenizer provides POS)
+    # - German: Compound word decomposition affects tagging
+    #
+    # Common POS tags (Penn Treebank style):
+    # - CC: Coordinating conjunction
+    # - CD: Cardinal number
+    # - DT: Determiner
+    # - EX: Existential there
+    # - FW: Foreign word
+    # - IN: Preposition or subordinating conjunction
+    # - JJ: Adjective
+    # - JJR: Adjective, comparative
+    # - JJS: Adjective, superlative
+    # - LS: List item marker
+    # - MD: Modal
+    # - NN: Noun, singular or mass
+    # - NNS: Noun, plural
+    # - NNP: Proper noun, singular
+    # - NNPS: Proper noun, plural
+    # - PDT: Predeterminer
+    # - POS: Possessive ending
+    # - PRP: Personal pronoun
+    # - PRP$: Possessive pronoun
+    # - RB: Adverb
+    # - RBR: Adverb, comparative
+    # - RBS: Adverb, superlative
+    # - RP: Particle
+    # - SYM: Symbol
+    # - TO: to
+    # - UH: Interjection
+    # - VB: Verb, base form
+    # - VBD: Verb, past tense
+    # - VBG: Verb, gerund or present participle
+    # - VBN: Verb, past participle
+    # - VBP: Verb, non-3rd person singular present
+    # - VBZ: Verb, 3rd person singular present
+    # - WDT: Wh-determiner
+    # - WP: Wh-pronoun
+    # - WP$: Possessive wh-pronoun
+    # - WRB: Wh-adverb
+    #
+    # Language-specific tags:
+    # - CJK uses its own tagset (e.g., Japanese: 名詞, 動詞, etc.)
+    # - German uses STTS tagset
+    #
+    # @abstract Subclasses must implement #tag
+    #
+    # @example Tagging tokens
+    #   tagger = EnglishPosTagger.new(aff_path: "en_US.aff", dic_path: "en_US.dic")
+    #   tokens = [
+    #     { token: "The", position: 0, length: 3 },
+    #     { token: "dog", position: 4, length: 3 },
+    #     { token: "runs", position: 8, length: 4 }
+    #   ]
+    #   tagged = tagger.tag(tokens)
+    #   # => [
+    #   #      { token: "The", position: 0, length: 3, pos_tag: "DET", lemma: "the" },
+    #   #      { token: "dog", position: 4, length: 3, pos_tag: "NOUN", lemma: "dog" },
+    #   #      { token: "runs", position: 8, length: 4, pos_tag: "VERB", lemma: "run" }
+    #   #    ]
+    class PosTagger
+      # Tag tokens with POS information.
+      #
+      # Takes an array of token hashes (from Tokenizer#tokenize) and adds:
+      # - :pos_tag (String, nil) - POS category (NOUN, VERB, etc.) or nil if unknown
+      # - :lemma (String, nil) - Lemma/base form or nil if unknown
+      #
+      # @abstract Subclasses must implement
+      # @param tokens [Array<Hash>] Array of token hashes from Tokenizer
+      # @return [Array<Hash>] Token hashes with added :pos_tag and :lemma keys
+      # @raise [NotImplementedError] if not implemented by subclass
+      def tag(tokens)
+        raise NotImplementedError, "#{self.class} must implement #tag"
+      end
+
+      # Tag a single word.
+      #
+      # Convenience method for single-word tagging.
+      #
+      # @param word [String] The word to tag
+      # @return [Hash] Hash with :pos_tag and :lemma keys (may be nil)
+      def tag_word(word)
+        token = { token: word, position: 0, length: word.length }
+        result = tag([token])
+        result.first || { pos_tag: nil, lemma: nil }
+      end
+    end
+  end
+end

data/lib/kotoshu/components/spell_checker.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Components
+    # Base class for spell checkers.
+    #
+    # Spell checkers validate words and provide suggestions for misspelled words.
+    # Different languages use different spell checking strategies:
+    # - Latin scripts: Dictionary lookup (Hunspell, Morfologik)
+    # - CJK: Confusion rule checking (no dictionary)
+    # - RTL: Dictionary lookup with bidirectional text handling
+    #
+    # @abstract Subclasses must implement #check and #suggest
+    #
+    # @example Checking a word
+    #   checker = EnglishSpellChecker.new(aff_path: "en_US.aff", dic_path: "en_US.dic")
+    #   result = checker.check("hello")
+    #   # => { found: true, stem: "hello", flags: [] }
+    #
+    # @example Getting suggestions
+    #   result = checker.check("helo")
+    #   # => { found: false, stem: nil, flags: [] }
+    #   suggestions = checker.suggest("helo")
+    #   # => [
+    #   #      { word: "hello", distance: 1, score: 0.9 },
+    #   #      { word: "help", distance: 2, score: 0.7 }
+    #   #    ]
+    class SpellChecker
+      # Check if a word is spelled correctly.
+      #
+      # Returns a hash with:
+      # - :found (Boolean) - true if word is in dictionary
+      # - :stem (String, nil) - The stem/lemma if found
+      # - :flags (Array<String>) - Morphological flags
+      #
+      # @abstract Subclasses must implement
+      # @param word [String] The word to check
+      # @return [Hash] Result with :found, :stem, :flags
+      # @raise [NotImplementedError] if not implemented by subclass
+      def check(word)
+        raise NotImplementedError, "#{self.class} must implement #check"
+      end
+
+      # Get spelling suggestions for a misspelled word.
+      #
+      # Returns an array of suggestion hashes with:
+      # - :word (String) - The suggested word
+      # - :distance (Integer) - Edit distance from original word
+      # - :score (Float) - Confidence score (0-1, higher is better)
+      #
+      # Suggestions are sorted by relevance (highest score first).
+      #
+      # @abstract Subclasses must implement
+      # @param word [String] The misspelled word
+      # @param max_suggestions [Integer] Maximum number of suggestions to return
+      # @return [Array<Hash>] Array of suggestion hashes
+      # @raise [NotImplementedError] if not implemented by subclass
+      def suggest(word, max_suggestions: 10)
+        raise NotImplementedError, "#{self.class} must implement #suggest"
+      end
+
+      # Check if a word is spelled correctly.
+      #
+      # Convenience method that returns a boolean.
+      #
+      # @param word [String] The word to check
+      # @return [Boolean] true if word is correct
+      def correct?(word)
+        check(word)[:found]
+      end
+    end
+  end
+end

data/lib/kotoshu/components/synthesizer.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Components
+    # Base class for word form synthesizers.
+    #
+    # Synthesizers generate inflected forms from a lemma (base form).
+    # This is the inverse of lemmatization:
+    # - Lemmatization: "runs" → "run"
+    # - Synthesis: "run" → ["run", "runs", "running", "ran"]
+    #
+    # Different languages use different synthesis strategies:
+    # - Latin scripts: Hunspell affix rules
+    # - CJK: Not applicable (no inflection)
+    # - German: Compound word + affix synthesis
+    # - Finnish: Complex agglutinative patterns
+    #
+    # @abstract Subclasses must implement #synthesize
+    #
+    # @example Synthesizing English verb forms
+    #   synthesizer = EnglishSynthesizer.new(aff_path: "en_US.aff", dic_path: "en_US.dic")
+    #   forms = synthesizer.synthesize("run", "VERB")
+    #   # => ["run", "runs", "running", "ran"]
+    #
+    # @example Synthesizing with POS constraint
+    #   forms = synthesizer.synthesize("happy", "ADJ")
+    #   # => ["happy", "happier", "happiest"]
+    class Synthesizer
+      # Generate inflected forms of a word.
+      #
+      # Given a lemma (base form) and a POS tag, returns all possible
+      # inflected forms of that word.
+      #
+      # @abstract Subclasses must implement
+      # @param lemma [String] The base form (lemma)
+      # @param pos_tag [String] The POS tag to constrain generation
+      # @return [Array<String>] Array of inflected forms
+      # @raise [NotImplementedError] if not implemented by subclass
+      def synthesize(lemma, pos_tag)
+        raise NotImplementedError, "#{self.class} must implement #synthesize"
+      end
+
+      # Generate all inflected forms (all POS tags).
+      #
+      # Convenience method that generates forms for all possible POS tags.
+      #
+      # @param lemma [String] The base form (lemma)
+      # @return [Hash] Hash mapping POS tags to arrays of forms
+      def synthesize_all(lemma)
+        # Default implementation - subclasses can optimize
+        {
+          'NOUN' => synthesize(lemma, 'NOUN'),
+          'VERB' => synthesize(lemma, 'VERB'),
+          'ADJ' => synthesize(lemma, 'ADJ'),
+          'ADV' => synthesize(lemma, 'ADV')
+        }
+      end
+    end
+  end
+end

data/lib/kotoshu/components/tokenizer.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Components
+    # Base class for tokenizers.
+    #
+    # Tokenizers split text into individual tokens (words, punctuation).
+    # Different languages use different tokenization strategies:
+    # - Latin scripts: Whitespace + punctuation
+    # - CJK: Morphological analysis
+    # - German: Compound word splitting
+    # - RTL: Right-to-left text handling
+    #
+    # @abstract Subclasses must implement #tokenize
+    #
+    # @example Tokenizing English text
+    #   tokenizer = WhitespaceTokenizer.new
+    #   tokens = tokenizer.tokenize("Hello, world!")
+    #   # => [
+    #   #      { token: "Hello", position: 0, length: 5 },
+    #   #      { token: ",", position: 5, length: 1 },
+    #   #      { token: "world", position: 7, length: 5 },
+    #   #      { token: "!", position: 12, length: 1 }
+    #   #    ]
+    class Tokenizer
+      # Split text into tokens.
+      #
+      # Each token is a hash with:
+      # - :token (String) - The token text
+      # - :position (Integer) - Character position in original text
+      # - :length (Integer) - Token length in characters
+      #
+      # Additional keys may be added by subclasses:
+      # - :pos_tag (String) - Part of speech tag
+      # - :lemma (String) - Base form / lemma
+      # - :compound_part (Boolean) - Whether this is a compound word part
+      # - :script (Symbol) - Script type for multilingual text
+      #
+      # @abstract Subclasses must implement
+      # @param text [String] The input text
+      # @return [Array<Hash>] Array of token hashes
+      # @raise [NotImplementedError] if not implemented by subclass
+      def tokenize(text)
+        raise NotImplementedError, "#{self.class} must implement #tokenize"
+      end
+
+      # Tokenize and return just the token strings.
+      #
+      # Convenience method for when you only need the text content.
+      #
+      # @param text [String] The input text
+      # @return [Array<String>] Array of token strings
+      def tokenize_to_strings(text)
+        tokenize(text).map { |t| t[:token] }
+      end
+    end
+  end
+end

data/lib/kotoshu/components/whitespace_tokenizer.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require_relative 'tokenizer'
+
+module Kotoshu
+  module Components
+    # Whitespace-based tokenizer for Latin-script languages.
+    #
+    # Splits text on whitespace and separates punctuation.
+    # Suitable for languages with space-separated words (English, French, German, etc.).
+    #
+    # This is a simple tokenizer that works well for most Latin-script languages.
+    # For more advanced tokenization (contractions, compounds), use language-specific
+    # tokenizers.
+    #
+    # @example Basic tokenization
+    #   tokenizer = WhitespaceTokenizer.new
+    #   tokens = tokenizer.tokenize("Hello, world!")
+    #   # => [
+    #   #      { token: "Hello", position: 0, length: 5 },
+    #   #      { token: ",", position: 5, length: 1 },
+    #   #      { token: "world", position: 7, length: 5 },
+    #   #      { token: "!", position: 12, length: 1 }
+    #   #    ]
+    #
+    # @example Tokenizing to strings
+    #   tokenizer.tokenize_to_strings("Hello, world!")
+    #   # => ["Hello", ",", "world", "!"]
+    class WhitespaceTokenizer < Tokenizer
+      # Regex pattern for matching tokens (words or punctuation).
+      TOKEN_PATTERN = /[\w']+|[^\w\s]/.freeze
+
+      # Create a new whitespace tokenizer.
+      #
+      # @param pattern [Regexp] Optional custom token pattern
+      def initialize(pattern: TOKEN_PATTERN)
+        @pattern = pattern
+      end
+
+      # Split text into tokens.
+      #
+      # Each token is a hash with:
+      # - :token (String) - The token text
+      # - :position (Integer) - Character position in original text
+      # - :length (Integer) - Token length in characters
+      #
+      # @param text [String] The input text
+      # @return [Array<Hash>] Array of token hashes
+      def tokenize(text)
+        return [] if text.nil? || text.empty?
+
+        tokens = []
+        position = 0
+
+        # Find all matches
+        text.scan(@pattern) do |match|
+          match_str = match.is_a?(Array) ? match.first : match
+          start_pos = text.index(match_str, position)
+
+          tokens << {
+            token: match_str,
+            position: start_pos,
+            length: match_str.length
+          }
+
+          position = start_pos + match_str.length
+        end
+
+        tokens
+      end
+
+      # Get the token pattern used by this tokenizer.
+      #
+      # @return [Regexp] The token pattern
+      def pattern
+        @pattern
+      end
+
+      # Check if a character is a word character.
+      #
+      # @param char [String] Single character
+      # @return [Boolean] True if word character
+      def word_char?(char)
+        char.match?(/[\w]/)
+      end
+
+      # Check if a character is punctuation.
+      #
+      # @param char [String] Single character
+      # @return [Boolean] True if punctuation
+      def punctuation?(char)
+        char.match?(/[^\w\s]/)
+      end
+    end
+  end
+end

data/lib/kotoshu/configuration/builder.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+require_relative "../configuration"
+
+module Kotoshu
+  class Configuration
+    # Builder for creating immutable Configuration objects.
+    #
+    # Provides a fluent interface for building configuration objects
+    # that are frozen after creation, ensuring thread-safety and immutability.
+    #
+    # @example Building with block
+    #   config = Configuration::Builder.build do |b|
+    #     b.dictionary_path = "words.txt"
+    #     b.language = "en-GB"
+    #   end
+    #
+    # @example Building with fluent methods
+    #   config = Configuration::Builder.build
+    #     .with_dictionary_path("words.txt")
+    #     .with_language("en-GB")
+    class Builder
+      # Build an immutable configuration.
+      #
+      # @yield [builder] Optional block for configuration
+      # @return [Configuration] Frozen configuration object
+      #
+      # @example With block
+      #   config = Builder.build do |b|
+      #     b.dictionary_path = "words.txt"
+      #   end
+      #
+      # @example Without block (uses defaults)
+      #   config = Builder.build
+      def self.build
+        builder_instance = new
+        yield(builder_instance) if block_given?
+        builder_instance.to_config
+      end
+
+      # Create a new builder.
+      def initialize
+        @settings = DEFAULTS.dup
+      end
+
+      # Set dictionary path.
+      #
+      # @param path [String] Path to dictionary file
+      # @return [self] Self for chaining
+      def dictionary_path=(path)
+        @settings[:dictionary_path] = path
+        self
+      end
+
+      # Set dictionary type.
+      #
+      # @param type [Symbol] Dictionary type
+      # @return [self] Self for chaining
+      def dictionary_type=(type)
+        @settings[:dictionary_type] = type
+        self
+      end
+
+      # Set language code.
+      #
+      # @param lang [String] Language code
+      # @return [self] Self for chaining
+      def language=(lang)
+        @settings[:language] = lang
+        self
+      end
+
+      # Set locale.
+      #
+      # @param locale [String, nil] Locale
+      # @return [self] Self for chaining
+      def locale=(locale)
+        @settings[:locale] = locale
+        self
+      end
+
+      # Set max suggestions.
+      #
+      # @param max [Integer] Maximum suggestions
+      # @return [self] Self for chaining
+      def max_suggestions=(max)
+        @settings[:max_suggestions] = max
+        self
+      end
+
+      # Set case sensitivity.
+      #
+      # @param sensitive [Boolean] Case sensitive flag
+      # @return [self] Self for chaining
+      def case_sensitive=(sensitive)
+        @settings[:case_sensitive] = sensitive
+        self
+      end
+
+      # Set verbose mode.
+      #
+      # @param verbose [Boolean] Verbose flag
+      # @return [self] Self for chaining
+      def verbose=(verbose)
+        @settings[:verbose] = verbose
+        self
+      end
+
+      # Set suggestion algorithms.
+      #
+      # @param algorithms [Array<Class>, nil] Suggestion algorithms
+      # @return [self] Self for chaining
+      def suggestion_algorithms=(algorithms)
+        @settings[:suggestion_algorithms] = algorithms
+        self
+      end
+
+      # Set custom words.
+      #
+      # @param words [Array<String>] Custom words
+      # @return [self] Self for chaining
+      def custom_words=(words)
+        @settings[:custom_words] = words.dup.freeze
+        self
+      end
+
+      # Set encoding.
+      #
+      # @param encoding [String] Character encoding
+      # @return [self] Self for chaining
+      def encoding=(encoding)
+        @settings[:encoding] = encoding
+        self
+      end
+
+      # Fluent method to set dictionary path.
+      #
+      # @param path [String] Path to dictionary file
+      # @return [Configuration] New configuration
+      def with_dictionary_path(path)
+        @settings[:dictionary_path] = path
+        self
+      end
+
+      # Fluent method to set dictionary type.
+      #
+      # @param type [Symbol] Dictionary type
+      # @return [Configuration] New configuration
+      def with_dictionary_type(type)
+        @settings[:dictionary_type] = type
+        self
+      end
+
+      # Fluent method to set language.
+      #
+      # @param lang [String] Language code
+      # @return [Configuration] New configuration
+      def with_language(lang)
+        @settings[:language] = lang
+        self
+      end
+
+      # Fluent method to set locale.
+      #
+      # @param locale [String, nil] Locale
+      # @return [Configuration] New configuration
+      def with_locale(locale)
+        @settings[:locale] = locale
+        self
+      end
+
+      # Fluent method to set max suggestions.
+      #
+      # @param max [Integer] Maximum suggestions
+      # @return [Configuration] New configuration
+      def with_max_suggestions(max)
+        @settings[:max_suggestions] = max
+        self
+      end
+
+      # Fluent method to set case sensitivity.
+      #
+      # @param sensitive [Boolean] Case sensitive flag
+      # @return [Configuration] New configuration
+      def with_case_sensitive(sensitive)
+        @settings[:case_sensitive] = sensitive
+        self
+      end
+
+      # Fluent method to set verbose mode.
+      #
+      # @param verbose [Boolean] Verbose flag
+      # @return [Configuration] New configuration
+      def with_verbose(verbose)
+        @settings[:verbose] = verbose
+        self
+      end
+
+      # Convert builder to frozen Configuration.
+      #
+      # @return [Configuration] Frozen configuration object
+      def to_config
+        config = Configuration.new(@settings.dup)
+        config.freeze
+        config
+      end
+    end
+  end
+end