kotoshu 0.3.0

data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Language
+    module Tokenizer
+      # Tokenizer for Portuguese text.
+      #
+      # Ported from LanguageTool's PortugueseWordTokenizer.
+      #
+      # Handles:
+      # - Decimal comma between digits (3,14)
+      # - Dotted numbers (1.000.000)
+      # - Dates (01.01.2024, 2024-01-01)
+      # - Colons in time (12:25)
+      # - Hyphens with do-not-split list
+      # - Spaced decimals (2 000 000)
+      class PortugueseTokenizer < Base
+        # Portuguese word separators - most punctuation and whitespace
+        # Note: We protect special patterns before splitting
+        WORD_SEPARATORS = /[\s"()\[\]{}<>@€£\\$%‰‱ºªᵃᵒˢ|`~#^·]/.freeze
+
+        # Placeholder characters (using non-printing characters)
+        DECIMAL_COMMA_SUBST = "\uE001"
+        NON_BREAKING_SPACE_SUBST = "\uE002"
+        NON_BREAKING_DOT_SUBST = "\uE003"
+        NON_BREAKING_COLON_SUBST = "\uE004"
+
+        # Decimal comma between digits: 3,14
+        DECIMAL_COMMA_PATTERN = /(\d),(\d)/
+
+        # Dotted numbers: 1.000.000
+        DOTTED_NUMBERS_PATTERN = /(\d)\.(\d)/
+
+        # Colon in numbers (time): 12:25
+        COLON_NUMBERS_PATTERN = /(\d):(\d)/
+
+        # Date patterns: 01.01.2024, 2024-01-01
+        DATE_PATTERN = /(\d{2})\.(\d{2})\.(\d{4})|(\d{4})\.(\d{2})\.(\d{2})|(\d{4})-(\d{2})-(\d{2})/
+
+        # Spaced decimals: 2 000 000
+        SPACED_DECIMAL_PATTERN = /(?<=^|[\s(])\d{1,3}( \d{3})+(?:[,#{DECIMAL_COMMA_SUBST}#{NON_BREAKING_DOT_SUBST}]\d+)?(?=\D|$)/
+
+        # Do-not-split list (from LanguageTool)
+        DO_NOT_SPLIT = %w[
+          mers-cov mcgraw-hill sars-cov-2 sars-cov
+          ph-metre ph-metres anti-ivg anti-uv anti-vih al-qaïda
+        ].freeze
+
+        def tokenize(text)
+          return [] if text.nil? || text.strip.empty?
+
+          # Handle decimal commas
+          if text.include?(",")
+            text = text.gsub(DECIMAL_COMMA_PATTERN, "\\1#{DECIMAL_COMMA_SUBST}\\2")
+          end
+
+          # Handle dots in numbers and dates
+          if text.include?(".")
+            # Handle dates first (before dotted numbers to avoid conflicts)
+            text = text.gsub(DATE_PATTERN) do |match|
+              # match[0] is the full match, match[1-9] are the capture groups
+              if match[1] && match[2] && match[3] # DD.MM.YYYY
+                "#{match[1]}#{NON_BREAKING_DOT_SUBST}#{match[2]}#{NON_BREAKING_DOT_SUBST}#{match[3]}"
+              elsif match[4] && match[5] && match[6] # YYYY.MM.DD
+                "#{match[4]}#{NON_BREAKING_DOT_SUBST}#{match[5]}#{NON_BREAKING_DOT_SUBST}#{match[6]}"
+              elsif match[7] && match[8] && match[9] # YYYY-MM-DD (keep as-is)
+                match[0]
+              else
+                match[0]
+              end
+            end
+            text = text.gsub(DOTTED_NUMBERS_PATTERN, "\\1#{NON_BREAKING_DOT_SUBST}\\2")
+          end
+
+          # Handle spaced decimals: 2 000 000
+          text = handle_spaced_decimals(text)
+
+          # Handle colons in time: 12:25
+          if text.include?(":")
+            text = text.gsub(COLON_NUMBERS_PATTERN, "\\1#{NON_BREAKING_COLON_SUBST}\\2")
+          end
+
+          # Split on word boundaries
+          raw_tokens = text.split(WORD_SEPARATORS)
+
+          # Process each token
+          tokens = []
+          raw_tokens.each do |token|
+            next if token.empty?
+
+            # Restore placeholders
+            token = restore_placeholders(token)
+
+            # Handle hyphenated words
+            parts = words_to_add(token)
+            tokens.concat(parts)
+          end
+
+          # Filter and normalize
+          tokens
+            .map { |token| normalize(token) }
+            .reject { |token| skip_token?(token) }
+        end
+
+        protected
+
+        # Restore placeholders to original characters.
+        #
+        # @param token [String] Token with placeholders
+        # @return [String] Token with restored characters
+        def restore_placeholders(token)
+          token
+            .gsub(DECIMAL_COMMA_SUBST, ",")
+            .gsub(NON_BREAKING_COLON_SUBST, ":")
+            .gsub(NON_BREAKING_SPACE_SUBST, " ")
+            .gsub(NON_BREAKING_DOT_SUBST, ".")
+        end
+
+        # Split a word into tokens, handling hyphens.
+        #
+        # @param word [String] Word to split
+        # @return [Array<String>] Array of tokens
+        def words_to_add(word)
+          return [word] unless word.include?("-")
+
+          # Check do-not-split list
+          return [word] if DO_NOT_SPLIT.include?(word.downcase)
+
+          # For now, split on hyphens if not in do-not-split list
+          # Future: integrate with tagger for better handling
+          word.split("-", -1).flat_map do |part|
+            part.empty? ? ["-"] : [part]
+          end
+        end
+
+        def word_separators
+          WORD_SEPARATORS
+        end
+
+        private
+
+        # Handle spaced decimals: 2 000 000.
+        #
+        # @param text [String] Input text
+        # @return [String] Text with non-breaking spaces
+        def handle_spaced_decimals(text)
+          result = text
+          text.scan(SPACED_DECIMAL_PATTERN) do
+            match = Regexp.last_match
+            split_number = match[0]
+            split_number_adjusted = split_number.gsub(" ", NON_BREAKING_SPACE_SUBST)
+            split_number_adjusted = split_number_adjusted.gsub("\u00A0", NON_BREAKING_SPACE_SUBST)
+            result = result.sub(split_number, split_number_adjusted)
+          end
+          result
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Language
+    module Tokenizer
+      # Tokenizer for Russian text.
+      #
+      # Ported from LanguageTool's RussianWordTokenizer.
+      #
+      # Handles:
+      # - Apostrophe as word character
+      # - Dot as word character (for abbreviations)
+      # - Special abbreviations: б/у (second-hand), б/н (new)
+      # - Spaced dots: .. , .
+      class RussianTokenizer < Base
+        # Russian-specific word separators (exclude apostrophe and dot)
+        WORD_SEPARATORS = /[\s"()\[\]{}<>,;:!?\\\/|`~@#$%^&*+\-·]/.freeze
+
+        # Special abbreviations that should not be split
+        # Using non-printing characters as placeholders
+        ABBREVIATION_PLACEHOLDERS = {
+          "б/у" => "\u0001\u0001SOCR_BU\u0001\u0001",
+          "б/н" => "\u0001\u0001SOCR_BN\u0001\u0001"
+        }.freeze
+
+        # Reverse placeholders for restoration
+        PLACEHOLDER_RESTORE = {
+          "\u0001\u0001SOCR_BU\u0001\u0001" => "б/у",
+          "\u0001\u0001SOCR_BN\u0001\u0001" => "б/н",
+          "\u0001\u0001SP_DDOT_SP\u0001\u0001" => " .. ",
+          "\u0001\u0001SP_DOT_SP\u0001\u0001" => " . ",
+          "\u0001\u0001SP_DOT\u0001\u0001" => "."
+        }.freeze
+
+        def tokenize(text)
+          return [] if text.nil? || text.strip.empty?
+
+          # Replace abbreviations with placeholders
+          text = replace_abbreviations(text)
+
+          # Split on word boundaries
+          raw_tokens = text.split(WORD_SEPARATORS)
+
+          # Restore abbreviations and filter
+          raw_tokens
+            .map { |token| restore_abbreviations(token) }
+            .map { |token| normalize(token) }
+            .reject { |token| skip_token?(token) }
+        end
+
+        protected
+
+        def word_separators
+          WORD_SEPARATORS
+        end
+
+        private
+
+        # Replace special abbreviations with placeholders.
+        #
+        # @param text [String] Input text
+        # @return [String] Text with placeholders
+        def replace_abbreviations(text)
+          result = text
+          ABBREVIATION_PLACEHOLDERS.each do |abbr, placeholder|
+            result = result.gsub(abbr, placeholder)
+          end
+
+          # Handle spaced dots
+          result = result.gsub(" .. ", "\u0001\u0001SP_DDOT_SP\u0001\u0001")
+          result = result.gsub(" . ", "\u0001\u0001SP_DOT_SP\u0001\u0001")
+          result = result.gsub(" .", " \u0001\u0001SP_DOT\u0001\u0001")
+
+          # Restore spaced dots first, then single dot pattern
+          result = result.gsub("\u0001\u0001SP_DDOT_SP\u0001\u0001", " .. ")
+          result = result.gsub("\u0001\u0001SP_DOT_SP\u0001\u0001", " . ")
+
+          result
+        end
+
+        # Restore abbreviations from placeholders.
+        #
+        # @param text [String] Text with placeholders
+        # @return [String] Text with restored abbreviations
+        def restore_abbreviations(text)
+          result = text
+          PLACEHOLDER_RESTORE.each do |placeholder, original|
+            result = result.gsub(placeholder, original)
+          end
+          result
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Language
+    module Tokenizer
+      # Tokenizer for Spanish text.
+      #
+      # Ported from LanguageTool's SpanishWordTokenizer.
+      #
+      # Handles:
+      # - Decimal point between digits (3.14)
+      # - Decimal comma between digits (3,14)
+      # - Ordinals (1.º, 2.ª, 1.er, 1.os, 1.as)
+      # - Hyphens (with do-not-split list since no tagger)
+      # - Soft hyphens
+      # - Inverted punctuation (¡, ¿)
+      class SpanishTokenizer < Base
+        # Spanish word separators - most punctuation and whitespace
+        # Note: We need to handle decimals specially, so we protect them first
+        WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*·]/.freeze
+
+        # Decimal point between digits: 3.14
+        DECIMAL_POINT = /(\d)\.(\d)/
+
+        # Decimal comma between digits: 3,14
+        DECIMAL_COMMA = /(\d),(\d)/
+
+        # Ordinal patterns: 1.º, 2.ª, 1.er, 1.os, 1.as
+        ORDINAL = /\b(\d+)\.(º|ª|o|a|er|os|as)\b/
+
+        # Placeholders for special patterns
+        DECIMAL_POINT_PLACEHOLDER = "\uE101"
+        DECIMAL_COMMA_PLACEHOLDER = "\uE102"
+        ORDINAL_PLACEHOLDER = "\uE103"
+
+        # Soft hyphen
+        SOFT_HYPHEN = "\u00AD"
+
+        # Do-not-split list (from LanguageTool)
+        DO_NOT_SPLIT = %w[
+          mers-cov mcgraw-hill sars-cov-2 sars-cov
+          ph-metre ph-metres
+        ].freeze
+
+        def tokenize(text)
+          return [] if text.nil? || text.strip.empty?
+
+          # Replace hyphen variants
+          text = text.gsub("\u2010", "\u002d")  # hyphen to hyphen-minus
+          text = text.gsub("\u2011", "\u002d")  # non-breaking hyphen to hyphen-minus
+
+          # Protect decimal points
+          text = text.gsub(DECIMAL_POINT, "\\1#{DECIMAL_POINT_PLACEHOLDER}\\2")
+
+          # Protect decimal commas
+          text = text.gsub(DECIMAL_COMMA, "\\1#{DECIMAL_COMMA_PLACEHOLDER}\\2")
+
+          # Protect ordinals
+          text = text.gsub(ORDINAL, "\\1#{ORDINAL_PLACEHOLDER}\\2")
+
+          # Split on word boundaries
+          raw_tokens = text.split(WORD_SEPARATORS)
+
+          # Process each token
+          tokens = []
+          raw_tokens.each do |token|
+            next if token.empty?
+
+            # Restore placeholders
+            token = restore_placeholders(token)
+
+            # Handle hyphenated words
+            parts = words_to_add(token)
+            tokens.concat(parts)
+          end
+
+          # Filter and normalize
+          tokens
+            .map { |token| normalize(token) }
+            .reject { |token| skip_token?(token) }
+        end
+
+        protected
+
+        # Restore placeholders to original characters.
+        #
+        # @param token [String] Token with placeholders
+        # @return [String] Token with restored characters
+        def restore_placeholders(token)
+          token
+            .gsub(DECIMAL_POINT_PLACEHOLDER, ".")
+            .gsub(DECIMAL_COMMA_PLACEHOLDER, ",")
+            .gsub(ORDINAL_PLACEHOLDER, ".")
+        end
+
+        # Split a word into tokens, handling hyphens.
+        #
+        # @param word [String] Word to split
+        # @return [Array<String>] Array of tokens
+        def words_to_add(word)
+          return [word] unless word.include?("-")
+
+          # Check do-not-split list
+          return [word] if DO_NOT_SPLIT.include?(word.downcase)
+
+          # Remove soft hyphens and check
+          normalized = word.gsub(SOFT_HYPHEN, "").gsub("'", "'")
+
+          # For now, split on hyphens if not in do-not-split list
+          # Future: integrate with tagger for better handling
+          normalized.split("-", -1).flat_map do |part|
+            part.empty? ? ["-"] : [part]
+          end
+        end
+
+        def word_separators
+          WORD_SEPARATORS
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/language.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require_relative "language/registry"
+require_relative "language/detector"
+require_relative "language/tokenizer/base"
+require_relative "language/tokenizer/latin_tokenizer"
+require_relative "language/tokenizer/french_tokenizer"
+require_relative "language/tokenizer/german_tokenizer"
+require_relative "language/tokenizer/spanish_tokenizer"
+require_relative "language/tokenizer/portuguese_tokenizer"
+require_relative "language/tokenizer/russian_tokenizer"
+require_relative "language/tokenizer/japanese_tokenizer"
+require_relative "language/normalizer/base"
+require_relative "language/languages/base"
+
+# Load all language-specific modules from new structure (languages/{en,fr,de,ja,pt,ru,es}/)
+require_relative "languages"
+
+module Kotoshu
+  # Language module for multi-language support.
+  #
+  # Provides language detection, tokenization, and normalization
+  # for different languages with proper OOP design.
+  #
+  # @example Detect language
+  #   Kotoshu::Language.detect("Hello world")  # => "en"
+  #
+  # @example Get language class
+  #   lang_class = Kotoshu::Language.get("en-US")
+  #
+  # @example List supported languages
+  #   Kotoshu::Language.supported_codes  # => ["de-DE", "en-US", ...]
+  module Language
+    # Register the default detector with the registry
+    Registry.register_detector(Detector)
+
+    class << self
+      # Detect language from text.
+      #
+      # Delegates to Detector.
+      #
+      # @param text [String] Text to analyze
+      # @return [String, nil] Detected language code
+      def detect(text)
+        Detector.detect(text)
+      end
+
+      # Detect with confidence score.
+      #
+      # @param text [String] Text to analyze
+      # @return [Array<String, Float>] Language code and confidence
+      def detect_with_confidence(text)
+        Detector.detect_with_confidence(text)
+      end
+
+      # Get language class by code.
+      #
+      # Delegates to Registry.
+      #
+      # @param code [String] Language code
+      # @return [Class, nil] Language class or nil
+      def get(code)
+        Registry.get(code)
+      end
+
+      # Check if language is registered.
+      #
+      # @param code [String] Language code
+      # @return [Boolean] True if registered
+      def registered?(code)
+        Registry.registered?(code)
+      end
+
+      # Get all supported language codes.
+      #
+      # @return [Array<String>] List of codes
+      def supported_codes
+        Registry.supported_codes
+      end
+
+      # Get language info.
+      #
+      # @param code [String] Language code
+      # @return [Hash, nil] Language info or nil
+      def info(code)
+        Registry.info(code)
+      end
+
+      # Register a language.
+      #
+      # @param code [String] Language code
+      # @param klass [Class] Language class
+      # @return [void]
+      def register(code, klass)
+        Registry.register(code, klass)
+      end
+    end
+  end
+end