kotoshu 0.3.0

data/lib/kotoshu/language/normalizer/base.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Language
+    module Normalizer
+      # Abstract base class for text normalizers.
+      #
+      # Normalizers transform text to a standard form for comparison.
+      # Different languages use different normalization strategies.
+      #
+      # Examples of normalization:
+      # - Accent removal (café -> cafe)
+      # - Case folding (Hello -> hello)
+      # - Whitespace normalization
+      # - Punctuation normalization
+      #
+      # @example Implement a normalizer
+      #   class MyNormalizer < Normalizer::Base
+      #     def normalize(text)
+      #       super.downcase.gsub(/[áàâä]/, 'a')
+      #     end
+      #   end
+      class Base
+        # Normalize text.
+        #
+        # Default implementation:
+        # - Strip leading/trailing whitespace
+        # - Collapse multiple whitespace to single space
+        # - Downcase (optional)
+        #
+        # @param text [String] Text to normalize
+        # @param options [Hash] Normalization options
+        # @option options [Boolean] :downcase (true) Convert to lowercase
+        # @option options [Boolean] :strip_punct (false) Remove punctuation
+        # @option options [Boolean] :collapse_ws (true) Collapse whitespace
+        # @return [String] Normalized text
+        def normalize(text, options = {})
+          return "" if text.nil?
+
+          defaults = {
+            downcase: true,
+            strip_punct: false,
+            collapse_ws: true
+          }
+          opts = defaults.merge(options)
+
+          result = text.dup
+
+          # Strip whitespace
+          result = result.strip
+
+          # Collapse multiple whitespace
+          result = result.gsub(/\s+/, " ") if opts[:collapse_ws]
+
+          # Downcase
+          result = result.downcase if opts[:downcase]
+
+          # Strip punctuation
+          result = strip_punctuation(result) if opts[:strip_punct]
+
+          result
+        end
+
+        # Normalize a word.
+        #
+        # @param word [String] Word to normalize
+        # @return [String] Normalized word
+        def normalize_word(word)
+          normalize(word)
+        end
+
+        # Check if two normalized strings are equal.
+        #
+        # @param str1 [String] First string
+        # @param str2 [String] Second string
+        # @return [Boolean] True if equal after normalization
+        def normalized_eql?(str1, str2)
+          normalize(str1) == normalize(str2)
+        end
+
+        protected
+
+        # Strip punctuation from text.
+        #
+        # @param text [String] Text to strip
+        # @return [String] Text without punctuation
+        def strip_punctuation(text)
+          text.gsub(/[^\p{L}\p{N}\s]/, "")
+        end
+
+        # Remove accents from characters.
+        #
+        # @param text [String] Text with accents
+        # @return [String] Text without accents
+        def remove_accents(text)
+          # Unicode normalization form D (decompose)
+          normalized = text.unicode_normalize(:nfd)
+
+          # Remove combining diacritical marks
+          normalized.gsub(/[\u0300-\u036F]/, "")
+        end
+
+        # Normalize quotes to standard ASCII.
+        #
+        # @param text [String] Text with quotes
+        # @return [String] Text with normalized quotes
+        def normalize_quotes(text)
+          # Left double quote to straight
+          text = text.gsub(/[\u201C\u201D]/, '"')
+          # Right double quote to straight
+          text = text.gsub(/[\u2018\u2019]/, "'")
+          # Backticks to quotes
+          text = text.gsub(/`/, "'")
+          # Other quote variants
+          text = text.gsub(/\u00AB/, '"')  # Left-pointing double angle
+          text = text.gsub(/\u00BB/, '"')  # Right-pointing double angle
+          text = text.gsub(/\u2039/, "'")  # Single left-pointing
+          text = text.gsub(/\u203A/, "'")  # Single right-pointing
+
+          text
+        end
+
+        # Normalize whitespace.
+        #
+        # @param text [String] Text with irregular whitespace
+        # @return [String] Text with normalized whitespace
+        def normalize_whitespace(text)
+          text
+            .gsub(/[\u00A0\u202F\u205F]/, " ")  # Various space chars
+            .gsub(/[\u2000-\u200B]/, " ")      # Various space chars
+            .gsub(/\s+/, " ")                   # Collapse multiple spaces
+            .strip
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/language/registry.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Language
+    # Central registry for language registration and retrieval.
+    #
+    # Uses Registry pattern for dynamic language discovery and management.
+    # Languages register themselves on load, making the system extensible.
+    #
+    # @example Register a language
+    #   Kotoshu::Language::Registry.register("en-US", English::American)
+    #
+    # @example Retrieve a language
+    #   lang = Kotoshu::Language::Registry.get("en-US")
+    #
+    # @example List supported languages
+    #   codes = Kotoshu::Language::Registry.supported_codes
+    class Registry
+      @languages = {}
+      @detectors = []
+
+      class << self
+        # Register a language class with its code.
+        #
+        # @param code [String] Language code (e.g., "en-US", "de-DE")
+        # @param language_class [Class] Class implementing Kotoshu::Language::Base
+        # @return [void]
+        #
+        # @example
+        #   Registry.register("en-US", English::American)
+        def register(code, language_class)
+          @languages[code] = language_class
+        end
+
+        # Register a detector for auto-detection.
+        #
+        # Detectors are tried in order of registration.
+        #
+        # @param detector [#detect] Object with detect method
+        # @return [void]
+        def register_detector(detector)
+          @detectors << detector
+        end
+
+        # Get language class by code.
+        #
+        # Supports fallback to base language if variant not found.
+        # Also supports finding variants when asking for base language.
+        # For example:
+        # - "en-GB" falls back to "en" if "en-GB" not registered
+        # - "en" returns "en-US" if only "en-US" is registered
+        #
+        # @param code [String] Language code
+        # @return [Class, nil] Language class or nil if not found
+        def get(code)
+          return nil unless code
+
+          # Try exact match first
+          return @languages[code] if @languages.key?(code)
+
+          base = code.split('-').first
+
+          # If code has a hyphen (e.g., "en-GB"), try base language
+          if code.include?('-')
+            return @languages[base]
+          end
+
+          # If code is base language (e.g., "en"), find any variant
+          @languages.each do |registered_code, klass|
+            return klass if registered_code.split('-').first == base
+          end
+
+          nil
+        end
+
+        # Check if a language is registered.
+        #
+        # @param code [String] Language code
+        # @return [Boolean] True if registered
+        def registered?(code)
+          !get(code).nil?
+        end
+
+        # Get all supported language codes.
+        #
+        # @return [Array<String>] Sorted list of language codes
+        def supported_codes
+          @languages.keys.sort
+        end
+
+        # Get all registered language classes.
+        #
+        # @return [Hash] Hash mapping codes to classes
+        def all
+          @languages.dup
+        end
+
+        # Detect language from text.
+        #
+        # Tries registered detectors in order.
+        #
+        # @param text [String] Text to analyze
+        # @return [String, nil] Detected language code or nil
+        def detect(text)
+          return nil if text.nil? || text.empty?
+
+          @detectors.each do |detector|
+            result = detector.detect(text)
+            return result if result
+          end
+
+          nil
+        end
+
+        # Clear all registrations (mainly for testing).
+        #
+        # @return [void]
+        def clear
+          @languages.clear
+          @detectors.clear
+        end
+
+        # Get language info by code.
+        #
+        # @param code [String] Language code
+        # @return [Hash, nil] Language info or nil
+        def info(code)
+          klass = get(code)
+          return nil unless klass
+
+          instance = klass.instance if klass.respond_to?(:instance)
+          instance ||= klass.new
+
+          {
+            code: code,
+            name: instance.name,
+            variant: instance.variant,
+            region: instance.region,
+            encoding: instance.encoding,
+            rtl?: instance.rtl?,
+            script_type: instance.script_type
+          }
+        end
+      end
+    end
+  end
+end