kotoshu 0.3.0

data/lib/kotoshu/language/resources/ru/confusion_sets.txt

@@ -0,0 +1,5 @@
+# Russian confusion sets
+# See the English 'confusion_sets.txt' for a description of file format
+
+не; ни; 1000000;                                   # p=1.000, r=0.550, 116+191, 3grams, 2016-10-13
+шасси; шоссе; 10000000;                            # p=1.000, r=0.125, 48+48, 3grams, 2016-10-12

data/lib/kotoshu/language/tokenizer/base.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Language
+    module Tokenizer
+      # Abstract base class for tokenizers.
+      #
+      # Uses Strategy pattern to allow different tokenization approaches
+      # for different languages.
+      #
+      # Subclasses must implement the tokenize method.
+      #
+      # @example Implement a tokenizer
+      #   class MyTokenizer < Tokenizer::Base
+      #     def tokenize(text)
+      #       text.split(/ /)
+      #     end
+      #   end
+      class Base
+        # Tokenize text into words.
+        #
+        # @param text [String] Text to tokenize
+        # @return [Array<String>] Array of tokens
+        # @raise [NotImplementedError] Must be implemented by subclass
+        def tokenize(text)
+          raise NotImplementedError, "#{self.class} must implement #tokenize"
+        end
+
+        # Tokenize text with positions.
+        #
+        # Returns tokens along with their position information.
+        #
+        # @param text [String] Text to tokenize
+        # @return [Array<Hash>] Array of {token:, start:, end:, line:, column:}
+        def tokenize_with_positions(text)
+          return [] if text.nil?
+          return [] if text.empty?
+
+          tokens = []
+          line = 1
+          column = 1
+          position = 0
+
+          while position < text.length
+            # Skip whitespace
+            while position < text.length && text[position].match?(/\s/)
+              if text[position] == "\n"
+                line += 1
+                column = 1
+              else
+                column += 1
+              end
+              position += 1
+            end
+
+            break if position >= text.length
+
+            # Find token
+            start_pos = position
+            start_line = line
+            start_column = column
+
+            token_text = extract_next_token(text, position)
+
+            if token_text
+              tokens << {
+                token: token_text,
+                start: start_pos,
+                end: start_pos + token_text.length,
+                line: start_line,
+                column: start_column
+              }
+
+              token_text.each_char do |char|
+                column += 1
+                position += 1
+                if char == "\n"
+                  line += 1
+                  column = 1
+                end
+              end
+            else
+              position += 1
+              column += 1
+            end
+          end
+
+          tokens
+        end
+
+        # Check if a character is a word character.
+        #
+        # @param char [String] Single character
+        # @return [Boolean] True if word character
+        def word_char?(char)
+          match?(word_boundary_regex, char)
+        end
+
+        # Get word boundary regex for this tokenizer.
+        #
+        # Subclasses should override this to define word boundaries.
+        #
+        # @return [Regexp] Word boundary regex
+        def word_boundary_regex
+          raise NotImplementedError, "#{self.class} must implement #word_boundary_regex"
+        end
+
+        # Normalize a token.
+        #
+        # Subclasses can override this for language-specific normalization.
+        #
+        # @param token [String] Token to normalize
+        # @return [String] Normalized token
+        def normalize(token)
+          token
+        end
+
+        # Check if a token should be skipped.
+        #
+        # Subclasses can override this for language-specific filtering.
+        #
+        # @param token [String] Token to check
+        # @return [Boolean] True if token should be skipped
+        def skip_token?(token)
+          return true if token.empty?
+          return true if token.match?(/^\d+$/) # Pure numbers
+          return true if token.length < 2 && token.match?(/^[^\p{L}]$/)
+
+          false
+        end
+
+        protected
+
+        # Extract the next token from text at position.
+        #
+        # @param text [String] Full text
+        # @param position [Integer] Current position
+        # @return [String, nil] Next token or nil
+        def extract_next_token(text, position)
+          remaining = text[position..]
+          match = remaining.match(/^#{word_pattern}/)
+          match ? match[0] : nil
+        end
+
+        # Get pattern for matching tokens.
+        #
+        # @return [String] Regex pattern string
+        def word_pattern
+          "[#{word_chars}]+"
+        end
+
+        # Get word characters for this tokenizer.
+        #
+        # @return [String] Character class of word characters
+        def word_chars
+          raise NotImplementedError, "#{self.class} must implement #word_chars"
+        end
+
+        # Check if string matches regex.
+        #
+        # @param regex [Regexp] Regex to match
+        # @param string [String] String to check
+        # @return [Boolean] True if matches
+        def match?(regex, string)
+          regex.match?(string)
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/language/tokenizer/french_tokenizer.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Language
+    module Tokenizer
+      # Tokenizer for French text.
+      #
+      # Ported from LanguageTool's FrenchWordTokenizer.
+      #
+      # Handles:
+      # - Apostrophes (l', d', qu', c'est, j'ai, etc.)
+      # - Hyphens (c'est-à-dire, rendez-vous, etc.)
+      # - Decimal points/commas
+      # - Multiple contraction patterns (7 total)
+      class FrenchTokenizer < Base
+        # French word separators - most punctuation and whitespace
+        # Note: apostrophe (') is NOT a separator in French (used for contractions)
+        WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*·]/.freeze
+
+        # Do-not-split list (from LanguageTool)
+        DO_NOT_SPLIT = %w[
+          mers-cov mcgraw-hill sars-cov-2 sars-cov
+          ph-metre ph-metres anti-ivg anti-uv anti-vih al-qaïda
+          c'est-à-dire add-on add-ons rendez-vous garde-à-vous
+          chez-eux chez-moi chez-nous chez-soi chez-toi chez-vous
+          m'as-tu-vu
+        ].freeze
+
+        # Contraction patterns (from LanguageTool)
+        # French contractions are complex: l', d', qu', c'est, j'ai, n'a, etc.
+        CONTRACTION_PATTERNS = [
+          # c' followed by word: c'est, c'était, etc.
+          /^(c[''])$/i,
+          # j' (je): j'ai, j'aime, etc.
+          /^(j[''])$/i,
+          # n' (ne): n'a, n'est, etc.
+          /^(n[''])$/i,
+          # m' (me): m'a, m'appelle, etc.
+          /^(m[''])$/i,
+          # t' (te): t'a, t'asseoir, etc.
+          /^(t[''])$/i,
+          # s' (se): s'a, s'appelle, etc.
+          /^(s[''])$/i,
+          # l' (le/la): l'a, l'homme, l'eau, etc.
+          /^(l[''])$/i,
+          # d' (de): d'un, d'une, d'abord, etc.
+          /^(d[''])$/i,
+          # qu' (que): qu'un, qu'une, qu'est, etc.
+          /^(qu[''])$/i,
+          # jusqu'à, jusqu'aux, etc.
+          /^(jusqu[''])$/i,
+          # puisque, puisqu'il, etc.
+          /^(puisqu[''])$/i,
+          # quoique, quoiqu'il, etc.
+          /^(quoiqu[''])$/i,
+          # lorsque, lorsqu'il, etc.
+          /^(lorsqu[''])$/i,
+        ].freeze
+
+        def tokenize(text)
+          return [] if text.nil? || text.strip.empty?
+
+          # Replace hyphen variants
+          text = text.gsub("\u2010", "\u002d")
+          text = text.gsub("\u2011", "\u002d")
+
+          # Normalize apostrophes
+          text = normalize_apostrophes(text)
+
+          # Split on word boundaries
+          raw_tokens = text.split(WORD_SEPARATORS)
+
+          # Process each token
+          tokens = []
+          raw_tokens.each do |token|
+            next if token.empty?
+
+            # Try to split contractions and hyphenated words
+            parts = split_french_word(token)
+            tokens.concat(parts)
+          end
+
+          # Filter and normalize
+          tokens
+            .map { |token| normalize(token) }
+            .reject { |token| skip_token?(token) }
+        end
+
+        protected
+
+        # Normalize apostrophes to straight quotes.
+        #
+        # @param text [String] Input text
+        # @return [String] Text with normalized apostrophes
+        def normalize_apostrophes(text)
+          text
+            .gsub("'", "'")
+            .gsub("'", "'")
+            .gsub("'", "'")
+        end
+
+        # Split French word, handling contractions and hyphens.
+        #
+        # @param word [String] Word to split
+        # @return [Array<String>] Array of tokens
+        def split_french_word(word)
+          # Check do-not-split list
+          return [word] if DO_NOT_SPLIT.include?(word.downcase)
+
+          # Handle hyphens first (but not for do-not-split words)
+          if word.include?("-")
+            # Check if it's a contraction pattern like "jusqu'à-ce"
+            if word.match?(/^(jusqu['']|[cç]['']|j['']|n['']|m['']|t['']|s['']|l['']|d['']|qu['']|lorsqu['']|puisqu['']|quoiqu[''])/)
+              # Split on hyphen for contractions
+              parts = []
+              word.split("-", -1).each do |part|
+                next if part.empty?
+                parts.concat(split_contractions(part))
+              end
+              return parts
+            else
+              # Regular hyphenated word - split it
+              return word.split("-", -1).reject(&:empty?)
+            end
+          end
+
+          # Handle contractions
+          if word.include?("'")
+            return split_contractions(word)
+          end
+
+          # No special handling needed
+          [word]
+        end
+
+        # Split contractions into component parts.
+        #
+        # @param word [String] Word that might be a contraction
+        # @return [Array<String>] Array of tokens
+        def split_contractions(word)
+          # Try each contraction pattern
+          CONTRACTION_PATTERNS.each do |pattern|
+            match = word.match(pattern)
+            if match
+              # Return the contraction and the rest of the word
+              contraction = match[1]
+              rest = word.sub(/^#{Regexp.escape(contraction)}/, "")
+              return [contraction, rest] unless rest.empty?
+              return [contraction]
+            end
+          end
+
+          # Handle special case: word starts with apostrophe
+          if word.match?(/^[cç]['']|^[a-z]['']/i)
+            # Split at the apostrophe
+            parts = word.split("'", 2)
+            return parts if parts.length == 2
+          end
+
+          # No pattern matched, return the word as-is
+          [word]
+        end
+
+        def word_separators
+          WORD_SEPARATORS
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/language/tokenizer/german_tokenizer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Language
+    module Tokenizer
+      # Tokenizer for German text.
+      #
+      # Ported from LanguageTool's GermanWordTokenizer.
+      #
+      # Handles:
+      # - Underscore as word character (not a separator)
+      # - Single low quote (‚) as word character (not a separator)
+      # - Umlauts (ä, ö, ü, ß)
+      #
+      # The LanguageTool implementation adds two characters to the word characters:
+      # underscore (_) and single low quote (‚ - U+201A).
+      class GermanTokenizer < Base
+        # German-specific word separators (exclude underscore and single low quote)
+        WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-·]/.freeze
+
+        def tokenize(text)
+          return [] if text.nil? || text.strip.empty?
+
+          # Split on word boundaries
+          raw_tokens = text.split(WORD_SEPARATORS)
+
+          # Filter and normalize
+          raw_tokens
+            .map { |token| normalize(token) }
+            .reject { |token| skip_token?(token) }
+        end
+
+        protected
+
+        def word_separators
+          WORD_SEPARATORS
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "suika"
+
+module Kotoshu
+  module Language
+    module Tokenizer
+      # Tokenizer for Japanese text.
+      #
+      # Uses Suika gem for morphological analysis.
+      #
+      # Suika is a pure Ruby Japanese morphological analyzer with a built-in
+      # dictionary from mecab-ipadic. It provides proper tokenization with
+      # part-of-speech information.
+      #
+      # @see https://github.com/yoshoku/suika
+      class JapaneseTokenizer < Base
+        # Japanese word separators - keep it simple since Suika handles tokenization
+        WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*·]/.freeze
+
+        # Class variable to hold the Suika tagger instance
+        @@tagger = nil
+
+        def tokenize(text)
+          return [] if text.nil? || text.strip.empty?
+
+          # Initialize tagger once (class variable for reuse)
+          @@tagger ||= ::Suika::Tagger.new
+
+          # Suika.parse returns an array of "surface\tfeatures" strings
+          tokens = []
+          parsed = @@tagger.parse(text)
+
+          parsed.each do |token|
+            # Suika returns: "すもも	名詞,一般,*,*,*,*,すもも,スモモ,スモモ"
+            # The surface form is tab-separated from the POS features
+            surface = token.split("\t").first
+            tokens << surface if surface && !surface.strip.empty?
+          end
+
+          tokens
+        end
+
+        protected
+
+        # Detect if text contains Japanese script.
+        #
+        # @param text [String] Text to check
+        # @return [Boolean] True if Japanese
+        def japanese?(text)
+          text.match?(/[\u3040-\u309F\u30A0-\u30FF]/) # Hiragana or Katakana
+        end
+
+        def word_separators
+          WORD_SEPARATORS
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Kotoshu
+  module Language
+    module Tokenizer
+      # Tokenizer for Latin-script languages.
+      #
+      # Base tokenizer for English, French, German, Spanish, Portuguese,
+      # and other European languages using Latin script.
+      #
+      # Handles:
+      # - Standard word boundaries (whitespace, punctuation)
+      # - Apostrophes within words (contractions, elisions)
+      # - Hyphenated words
+      # - Numbers with units
+      #
+      # Subclasses can override for language-specific handling.
+      class LatinTokenizer < Base
+        # Latin word characters including accented characters
+        WORD_CHARS = "a-zA-Zà-ÿ0-9'"
+
+        # Punctuation that separates words
+        WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-=_]/
+
+        # Contractions that should stay together
+        CONTRACTIONS = %w[
+          I'm I'd I've I'll you're you'd you've you'll he's he'd he'll
+          she's she'd she'll it's it'd we're we'd we've we'll they're
+          they'd they've they'll that's that'd that'll who's who'd who'll
+          what's what'd what'll where's where'd when's when'd why's why'd
+          how's how'd can't won't don't shouldn't couldn't wouldn't didn't
+          isn't aren't wasn't weren't hasn't haven't hadn't doesn't do
+          doesn't didn't mightn't mustn't shan't shouldn't wouldn't
+        ].freeze
+
+        # Tokenize text into words.
+        #
+        # @param text [String] Text to tokenize
+        # @return [Array<String>] Array of tokens
+        def tokenize(text)
+          return [] if text.nil? || text.strip.empty?
+
+          # Split on word boundaries
+          raw_tokens = text.split(WORD_SEPARATORS)
+
+          # Filter and normalize
+          raw_tokens
+            .map { |token| normalize(token) }
+            .reject { |token| skip_token?(token) }
+        end
+
+        # Get word boundary regex.
+        #
+        # @return [Regexp] Word boundary regex
+        def word_boundary_regex
+          /[#{WORD_CHARS}]/
+        end
+
+        # Normalize token.
+        #
+        # Subclasses can override for language-specific normalization.
+        #
+        # @param token [String] Token to normalize
+        # @return [String] Normalized token
+        def normalize(token)
+          token.strip
+        end
+
+        # Check if token should be skipped.
+        #
+        # @param token [String] Token to check
+        # @return [Boolean] True if should skip
+        def skip_token?(token)
+          return true if super
+
+          # Skip pure numbers
+          return true if token.match?(/^\d+$/)
+
+          # Skip single characters (unless a word)
+          return true if token.length == 1 && token.match?(/[^a-zA-Zà-ÿ]/)
+
+          # Skip empty tokens
+          return true if token.empty?
+
+          # Skip tokens with no letters
+          return true unless token.match?(/[a-zA-Zà-ÿ]/)
+
+          false
+        end
+
+        protected
+
+        # Get word characters.
+        #
+        # @return [String] Character class
+        def word_chars
+          WORD_CHARS
+        end
+
+        # Handle contractions to keep them together.
+        #
+        # @param text [String] Input text
+        # @return [String] Text with protected contractions
+        def handle_contractions(text)
+          result = text.dup
+
+          # Protect common contractions
+          CONTRACTIONS.each do |contraction|
+            # Use word boundaries to avoid partial matches
+            result = result.gsub(/\b#{Regexp.escape(contraction)}\b/, contraction.gsub("'", "\uFEFF"))
+          end
+
+          result
+        end
+
+        # Extract next token with position.
+        #
+        # Override to handle apostrophes within words.
+        #
+        # @param text [String] Full text
+        # @param position [Integer] Current position
+        # @return [String, nil] Next token or nil
+        def extract_next_token(text, position)
+          remaining = text[position..]
+
+          # Check for contraction first
+          CONTRACTIONS.each do |contraction|
+            if remaining.start_with?(contraction) &&
+               remaining[contraction.length]&.match?(/\s|[^a-zA-Zà-ÿ]/)
+              return contraction
+            end
+          end
+
+          # Extract word with potential apostrophe
+          match = remaining.match(/^([#{WORD_CHARS}]+(?:'[#{WORD_CHARS}]+)?)/)
+          match ? match[1] : nil
+        end
+      end
+    end
+  end
+end