pragmatic_tokenizer 3.0.3 → 3.1.0

data/lib/pragmatic_tokenizer/regex.rb

@@ -0,0 +1,149 @@
+module PragmaticTokenizer
+  class Regex
+
+    # Things that can or should be done:
+    # - check where the use of unicode categories helps (\p{Abbreviation})
+    # - use URI.parse and other libraries instead of regexp to identify urls, domains, emails
+    # - check multiple domain regex, we have spec issues when using one or the other
+    # - check multiple punctuation regex
+
+    # Text that needs to be tokenized is initially split into chunks of this length:
+    CHUNK_LONG_INPUT_TEXT         = /\S.{1,10000}(?!\S)/m
+
+    # Ranges
+    RANGE_DINGBATS                = /[\u2701-\u27BE]/ # e.g. ✁✎✳❄➾
+    RANGE_VARIATION_SELECTORS     = /[\uFE00-\uFE0F]/ # alter the previous character
+    RANGE_FULLWIDTH               = /[\uFF01-\ufF1F]/ # e.g. ！＂＃＇？
+    RANGE_ALPHANUMERIC_SUPPLEMENT = /[\u{1F100}-\u{1F1FF}]/
+    RANGE_UNUSUAL_AND_EMOJI       = /[\u203C-\u3299\u{1F000}-\u{1F644}]/
+
+    # Regular expressions which do not need to capture anything are enclosed in /(?: … )/ to enhance performance
+    COLON1                        = /(?:(:)([[:print:]]{2,}))/ # two non-space after colon prevent matching emoticons
+    COLON2                        = /(?::)/
+    COMMAS                        = /(?:([,‚])+)/
+    ENCLOSED_PLUS                 = /(?:([[:print:]]+)\+([[:print:]]+))/
+    EMAIL                         = /(?:[[:print:]]+[＠@][[:print:]]+\.[[:print:]]+)/
+    DIGIT                         = /(?:[[:digit:]]+)/
+    ASTERISK                      = /(?:\*+)/
+    UNDERSCORE                    = /(?:_+)/
+    HYPHEN_OR_UNDERSCORE          = /(?:[-_])/
+    PERIOD_AND_PRIOR              = /(?:(.+\.))/
+    PERIOD_ONLY                   = /(?:(\.))/
+    CONTRACTIONS                  = /(?:[‘’‚‛‹›＇´`])/
+    PUNCTUATION1                  = /(?:([\p{Pd}\p{Pe}\p{Pf}\p{Pi}\p{Ps}])+)/ # all punctuation categories except Pc (Connector) and Po (other)
+    PUNCTUATION2                  = /(?:(?<=\S)([!?#{RANGE_FULLWIDTH.source}]+))/
+    PUNCTUATION3                  = /(?:[!%\-–\u00AD]+)/
+    PUNCTUATION4                  = /(?:[.．。]+)/
+    DINGBATS                      = /(?:(#{RANGE_DINGBATS.source}#{RANGE_VARIATION_SELECTORS.source}*+))/
+    NO_BREAK_SPACE                = /(?:\u00A0+)/
+    HTTP                          = /(?:https?:\/\/)/
+    TIME_WITH_COLON               = /(?:\d:\d)/
+    DOMAIN_PREFIX                 = /(?:https?:\/\/|www\.|[[:alpha:]]\.)/
+    DOMAIN_SUFFIX                 = /(?:[[:alpha:]]\.(?:com|net|org|edu|gov|mil|int|[[:alpha:]]{2}))/
+    DOMAIN1                       = /(?:((https?:\/\/|)[[:print:]]+\.[[:alpha:]]{2,6}(:[0-9]{1,5})?(\/[[:print:]]*+)?))/
+    DOMAIN2                       = /(?:[[:alnum:]]{2,}([\-.][[:alnum:]]+)*\.[[:alpha:]]{2,6}(:[0-9]{1,5})?(\/[[:print:]]*+)?)/
+    NOT_URL                       = /(?:^(?!#{DOMAIN_PREFIX.source})([[:print:]]*))/
+    HASHTAG_OR_MENTION            = /(?:[@#＠＃][[:print:]]+)/
+    HASHTAG                       = /(?:[#＃][[:print:]]+)/
+    MENTION                       = /(?:[@＠][[:print:]]+)/
+    HASHTAG_WITH_HYPHEN           = /(?:^([#＃][[:digit:]]+)-)/
+    ONE_AS_EXCLAMATION            = /(?:\D1+)/
+    ONES_EXCLAMATIONS             = /(?:!+(1*+!*+)*+)/
+    MANY_PERIODS                  = /(?:^\.{2,}$)/
+    COPYRIGHT_TRADEMARK           = /(?:[®©™]+)/
+    CONTROL_CHARACTER             = /(?:[[:cntrl:]]+)/ # matches any character with hexadecimal value 00 through 1F or 7F.
+    APOSTROPHE_AND_S              = /(?:['’`́]s)/
+    ALSO_DECIMALS                 = /(?:[[:alpha:]]*+[[:digit:]]+)/
+    ACUTE_ACCENT_S                = /(?:\s\u0301(?=s))/
+
+    # Regular expressions used to capture items
+    CAPTURE_UNUSUAL_AND_EMOJI     = /(#{RANGE_UNUSUAL_AND_EMOJI.source})/
+    QUESTION_MARK_NOT_URL         = /#{NOT_URL.source}(\?)/
+    # Should we change specs and also capture "/", just like we capture ":" and "?"
+    SLASH_NOT_URL                 = /#{NOT_URL.source}\//
+    SHIFT_BOUNDARY_CHARACTERS     = /([;^&|…«»„“¿¡≠]+)/
+    MULTIPLE_DOTS                 = /(\.{2,})/ # we keep all dashes
+    MULTIPLE_DASHES               = /(-){2,}/ # we only keep first dash
+    BRACKET                       = /([{}()\[\]])/
+    EXCLAMATION_BETWEEN_ALPHA     = /(?<=[[:alpha:]])(!)(?=[[:alpha:]])/
+    PERCENT_BEFORE_DIGIT          = /(%)\d+/
+    COMMA_BEFORE_NON_DIGIT        = /(,)(?=\D)/
+    COMMA_AFTER_NON_DIGIT         = /(?<=\D)(,)/
+    COLON_IN_URL                  = /(?<=[(https?|ftp)]):(?=\/\/)/
+    QUOTE_BEFORE_PRINT            = /(('')|["“])(?=[[:print:]])/
+    QUOTE                         = /('')|["”]/
+    HYPHEN_AFTER_NON_WORD         = /(?<=\W)(-)/
+    HYPHEN_BEFORE_NON_WORD        = /(-)(?=\W)/
+
+    STARTS_WITH_COMMAS            = /^#{COMMAS.source}/
+    STARTS_WITH_HTTP              = /^#{HTTP.source}/
+    STARTS_WITH_DOMAIN            = /^#{DOMAIN_PREFIX.source}/
+    STARTS_WITH_COLON1            = /^#{COLON1.source}/
+    STARTS_WITH_UNDERSCORE        = /^#{UNDERSCORE.source}/
+    STARTS_WITH_PUNCTUATION3      = /^#{PUNCTUATION3.source}/
+
+    ENDS_WITH_DOMAIN              = /#{DOMAIN_SUFFIX.source}$/
+    ENDS_WITH_PUNCTUATION1        = /#{PUNCTUATION1.source}$/
+    ENDS_WITH_PUNCTUATION2        = /#{PUNCTUATION2.source}$/
+    ENDS_WITH_COLON2              = /#{COLON2.source}$/
+    ENDS_WITH_UNDERSCORE          = /#{UNDERSCORE.source}$/
+    ENDS_WITH_ONES_EXCLAMATIONS   = /#{ONES_EXCLAMATIONS.source}$/
+    ENDS_WITH_EXCITED_ONE         = /#{ONE_AS_EXCLAMATION.source}$/
+    ENDS_WITH_APOSTROPHE_AND_S    = /#{APOSTROPHE_AND_S.source}$/
+    ENDS_WITH_ALPHA               = /[[:alpha:]]$/
+    ENDS_WITH_DIGIT               = /[[:digit:]]$/
+
+    ONLY_DECIMALS                 = /(?:^[[:digit:]]+$)/
+    NO_DECIMALS                   = /(?:^\D+$)/
+    ONLY_PUNCTUATION              = /^[[[:punct:]]^|+]+$/
+    ONLY_ROMAN_NUMERALS           = /^(?=[MDCLXVI])M*(C[MD]|D?C*)(X[CL]|L?X*)(I[XV]|V?I*)$/i
+    ONLY_EMAIL                    = /^#{EMAIL}$/
+    ONLY_HASHTAG_MENTION          = /^#{HASHTAG_OR_MENTION}$/
+    ONLY_HASHTAG                  = /^#{HASHTAG}$/
+    ONLY_MENTION                  = /^#{MENTION}$/
+    ONLY_DOMAIN1                  = /^#{DOMAIN1}$/
+    ONLY_DOMAIN2                  = /^#{DOMAIN2}$/
+    ONLY_DOMAIN3                  = Regexp.union(STARTS_WITH_DOMAIN, ENDS_WITH_DOMAIN)
+    DOMAIN_OR_EMAIL               = Regexp.union(ONLY_DOMAIN1, ONLY_EMAIL)
+    UNDERSCORES_ASTERISK          = Regexp.union(STARTS_WITH_UNDERSCORE, ENDS_WITH_UNDERSCORE, ASTERISK)
+    NO_DECIMALS_NO_NUMERALS       = Regexp.union(ALSO_DECIMALS, ONLY_ROMAN_NUMERALS)
+
+    COMMAS_OR_PUNCTUATION = Regexp.union(
+        STARTS_WITH_COMMAS,
+        ENDS_WITH_PUNCTUATION1,
+        ENDS_WITH_PUNCTUATION2
+    )
+
+    # Can this constant name be clarified?
+    VARIOUS = Regexp.union(
+        SLASH_NOT_URL,
+        QUESTION_MARK_NOT_URL,
+        ENCLOSED_PLUS,
+        STARTS_WITH_COLON1,
+        DINGBATS,
+        HASHTAG_WITH_HYPHEN,
+        CAPTURE_UNUSUAL_AND_EMOJI
+    )
+
+    IRRELEVANT_CHARACTERS = Regexp.union(
+        STARTS_WITH_PUNCTUATION3,
+        ENDS_WITH_COLON2,
+        ENDS_WITH_ONES_EXCLAMATIONS,
+        CONTROL_CHARACTER,
+        COPYRIGHT_TRADEMARK,
+        RANGE_ALPHANUMERIC_SUPPLEMENT
+    )
+
+    PRE_PROCESS = Regexp.union(
+        SHIFT_BOUNDARY_CHARACTERS,
+        MULTIPLE_DOTS,
+        BRACKET,
+        MULTIPLE_DASHES,
+        EXCLAMATION_BETWEEN_ALPHA,
+        PERCENT_BEFORE_DIGIT,
+        COMMA_BEFORE_NON_DIGIT,
+        COMMA_AFTER_NON_DIGIT
+    )
+
+  end
+end

data/lib/pragmatic_tokenizer/tokenizer.rb

@@ -1,69 +1,22 @@
-# -*- encoding : utf-8 -*-
 require 'set'
 require 'cgi'
+require 'pragmatic_tokenizer/regex'
 require 'pragmatic_tokenizer/languages'
 require 'pragmatic_tokenizer/pre_processor'
 require 'pragmatic_tokenizer/post_processor'
-require 'pragmatic_tokenizer/full_stop_separator'
 require 'unicode'
 
 module PragmaticTokenizer
   class Tokenizer
 
-    PUNCTIATION_OPTIONS       = Set.new([:all, :semi, :none, :only]).freeze
-    NUMBERS_OPTIONS           = Set.new([:all, :semi, :none, :only]).freeze
-    MENTIONS_OPTIONS          = Set.new([:keep_original, :keep_and_clean, :remove]).freeze
+    PUNCTUATION_OPTIONS       = Set.new(%i[all semi none only]).freeze
+    NUMBERS_OPTIONS           = Set.new(%i[all semi none only]).freeze
+    MENTIONS_OPTIONS          = Set.new(%i[keep_original keep_and_clean remove]).freeze
     MAX_TOKEN_LENGTH          = 50
-    EMPTY_STRING              = ''.freeze
-    DOT_STRING                = '.'.freeze
-    SPACE_STRING              = ' '.freeze
-    REGEX_DOMAIN              = /(\s+|\A)[a-z0-9]{2,}([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,6}(:[0-9]{1,5})?(\/.*)?/ix
-    REGEX_URL                 = /(http|https)(\.|:)/
-    REGEX_HYPHEN              = /\-/
-    REGEX_UNDERSCORE          = /\_/
-    REGEX_CONTRACTIONS        = /[‘’‚‛‹›＇´`]/
-    REGEX_APOSTROPHE_S        = /['’`́]s$/
-    REGEX_EMAIL               = /\S+(＠|@)\S+\.\S+/
-    REGEX_HASHTAG_OR_MENTION  = /[＠@#|＃]/
-    REGEX_UNDERSCORE_AT_START = /(?<=\A)\_+/
-    REGEX_UNDERSCORE_AT_END   = /\_+(?=\z)/
-    REGEX_ASTERISK            = /\*+/
-    REGEX_UNIFIED1            = Regexp.union(REGEX_UNDERSCORE_AT_START,
-                                             REGEX_UNDERSCORE_AT_END,
-                                             REGEX_ASTERISK)
-    # https://en.wikipedia.org/wiki/Control_character
-    # matches any character with hexadecimal value 00 through 1F or 7F.
-    # Rubular: http://rubular.com/r/E83fpBoDjI
-    REGEXP_CONTROL                  = /[[:cntrl:]]/
-    REGEXP_ENDING_COLON             = /\:(?=\z)/
-    REGEXP_EXCLAMATION_AT_START     = /(?<=\A)!+(?=.+)/
-    REGEXP_EXCLAMATION_AT_END       = /!+(1*!*)*(?=\z)/
-    REGEXP_HYPHEN_AT_START          = /\A(-|–|\u{00AD})/
-    REGEXP_SPECIAL_SYMBOL           = /[®©]/
-    REGEXP_PERCENT_AT_START         = /\A\%/
-    # https://codepoints.net/enclosed_alphanumeric_supplement
-    REGEXP_ALPHANUMERIC_SUPPLEMENT  = /[\u{1F100}-\u{1F1FF}]/
-    REGEX_UNIFIED2                  = Regexp.union(REGEXP_CONTROL,
-                                                   REGEXP_ENDING_COLON,
-                                                   REGEXP_EXCLAMATION_AT_START,
-                                                   REGEXP_EXCLAMATION_AT_END,
-                                                   REGEXP_HYPHEN_AT_START,
-                                                   REGEXP_SPECIAL_SYMBOL,
-                                                   REGEXP_PERCENT_AT_START,
-                                                   REGEXP_ALPHANUMERIC_SUPPLEMENT)
-    REGEXP_ONE_AS_EXCLAMATION  = /(?<=\D)1+(?=\z)/
-    REGEXP_HASHTAG_AT_START    = /(?<=\A)(#|＃)/
-    REGEXP_AT_SIGN_AT_START    = /(?<=\A)(@|＠)/
-    REGEXP_HYPHEN_HASTAG       = /\A(#|＃)\S+-/
-    REGEXP_EMOJI_SNOWFLAKE     = /\u{2744}[\u{FE0F}|\u{FE0E}]?/
-    REGEX_EMOJI_UNIFIED        = Regexp.union(REGEXP_EMOJI_SNOWFLAKE,
-                                            PragmaticTokenizer::Languages::Common::EMOJI_REGEX)
-    REGEXP_PUNCTUATION_ONLY    = /\A[[:punct:]]+\z/
-    REGEXP_NUMBER_ONLY         = /\A\d+\z/
-    REGEXP_NO_NUMBERS          = /\A\D+\z/
-    REGEXP_NUMBER              = /\D*\d+\d*/
-    REGEXP_CONSECUTIVE_DOTS    = /\A\.{2,}\z/
-    REGEXP_CHUNK_STRING        = /.{,10000}(?=\s|\z)/m
+    NOTHING                   = ''.freeze
+    DOT                       = '.'.freeze
+    SPACE                     = ' '.freeze
+    SINGLE_QUOTE              = "'".freeze
 
     # @param [Hash] opts optional arguments
 
@@ -123,10 +76,10 @@ module PragmaticTokenizer
       @abbreviations       = Set.new(opts[:abbreviations])
       @stop_words          = Set.new(opts[:stop_words])
 
-      # TODO: why do we treat stop words differently than abbreviations and contractions? (we don't use @language_module::STOP_WORDS when passing @filter_languages)
+      # Why do we treat stop words differently than abbreviations and contractions? (we don't use @language_module::STOP_WORDS when passing @filter_languages)
       @contractions.merge!(@language_module::CONTRACTIONS) if @contractions.empty?
       @abbreviations       += @language_module::ABBREVIATIONS if @abbreviations.empty?
-      @stop_words          += @language_module::STOP_WORDS if @stop_words.empty? && @filter_languages.empty?
+      @stop_words          += @language_module::STOP_WORDS if @stop_words.empty?
 
       @filter_languages.each do |lang|
         language = Languages.get_language_by_code(lang)
@@ -135,34 +88,43 @@ module PragmaticTokenizer
         @stop_words    += language::STOP_WORDS
       end
 
-      raise "Punctuation argument can be only be nil, :all, :semi, :none, or :only" unless PUNCTIATION_OPTIONS.include?(@punctuation)
+      raise "Punctuation argument can be only be nil, :all, :semi, :none, or :only" unless PUNCTUATION_OPTIONS.include?(@punctuation)
       raise "Numbers argument can be only be nil, :all, :semi, :none, or :only" unless NUMBERS_OPTIONS.include?(@numbers)
       raise "Mentions argument can be only be nil, :keep_original, :keep_and_clean, or :remove" unless MENTIONS_OPTIONS.include?(@mentions)
-      raise "In Pragmatic Tokenizer minimum_length must be an Integer" unless @minimum_length.class == Fixnum || @minimum_length.nil?
-      raise "In Pragmatic Tokenizer long_word_split must be an Integer" unless @long_word_split.class == Fixnum || @long_word_split.nil?
+
+      integer_class = Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.4.0') ? Fixnum : Integer
+
+      raise "In Pragmatic Tokenizer minimum_length must be an Integer"  unless @minimum_length.class  == integer_class || @minimum_length.nil?
+      raise "In Pragmatic Tokenizer long_word_split must be an Integer" unless @long_word_split.class == integer_class || @long_word_split.nil?
     end
 
     # @param [String] text to be tokenized
 
     def tokenize(text)
       return [] unless text
-      raise "In Pragmatic Tokenizer text must be a String" unless text.class == String
+      raise "In PragmaticTokenizer text must be a String or subclass of String" unless text.class <= String
       CGI.unescapeHTML(text)
-          .scan(REGEXP_CHUNK_STRING)
-          .flat_map { |segment| post_process(pre_process(segment)) }
+          .scan(Regex::CHUNK_LONG_INPUT_TEXT)
+          .flat_map { |segment| process_segment(segment) }
     end
 
     private
 
-      def pre_process(text)
-        text
+      def process_segment(segment)
+        pre_processed = pre_process(segment)
+        cased_segment = chosen_case(pre_processed)
+        @tokens       = PostProcessor.new(text: cased_segment, abbreviations: @abbreviations, downcase: @downcase).call
+        post_process_tokens
+      end
+
+      def pre_process(segment)
+        segment
             .extend(PragmaticTokenizer::PreProcessor)
             .pre_process(language: @language_module)
       end
 
-      def post_process(text)
-        @tokens = run_post_processor(text)
-        remove_various!
+      def post_process_tokens
+        remove_by_options!
         process_numbers!
         process_punctuation!
         expand_contractions! if @expand_contractions
@@ -176,45 +138,45 @@ module PragmaticTokenizer
         @tokens.reject(&:empty?)
       end
 
-      def run_post_processor(text)
-        PostProcessor.new(
-            text:          chosen_case(text),
-            abbreviations: @abbreviations,
-            downcase:      @downcase
-        ).post_process
-      end
-
       def expand_contractions!
-        @tokens = @tokens.flat_map { |t| expand_token_contraction(t) }
+        @tokens = @tokens.flat_map { |token| expand_token_contraction(token) }
       end
 
       def expand_token_contraction(token)
-        normalized = inverse_case(token.gsub(REGEX_CONTRACTIONS, "'".freeze))
+        normalized = inverse_case(token.gsub(Regex::CONTRACTIONS, SINGLE_QUOTE))
         return token unless @contractions.key?(normalized)
-        result    = @contractions[normalized].split(SPACE_STRING)
+        result    = @contractions[normalized].split(SPACE)
         result[0] = Unicode.capitalize(result[0]) unless @downcase
         result
       end
 
       def clean!
         @tokens = @tokens
-            .flat_map { |t| t !~ REGEX_HASHTAG_OR_MENTION ? t.split(REGEX_UNIFIED1) : t }
-            .map! { |t| t !~ REGEX_HASHTAG_OR_MENTION ? t.gsub(REGEXP_ONE_AS_EXCLAMATION, EMPTY_STRING) : t }
-            .map! { |t| t.gsub(REGEX_UNIFIED2, EMPTY_STRING) }
-            .delete_if { |t| unclean_token?(t) }
+            .flat_map  { |token| split_underscores_asterisk(token) }
+            .map!      { |token| remove_irrelevant_characters(token) }
+            .delete_if { |token| many_dots?(token) }
       end
 
-      def unclean_token?(token)
-        return true if PragmaticTokenizer::Languages::Common::SPECIAL_CHARACTERS.include?(token)
-        return true if token.length > MAX_TOKEN_LENGTH
-        return true if token.include?('\\'.freeze)
-        token =~ REGEXP_CONSECUTIVE_DOTS
+      def split_underscores_asterisk(token)
+        return token if token =~ Regex::ONLY_HASHTAG_MENTION
+        token.split(Regex::UNDERSCORES_ASTERISK)
+      end
+
+      def remove_irrelevant_characters(token)
+        token.gsub!(Regex::IRRELEVANT_CHARACTERS, NOTHING)
+        return token if token =~ Regex::ONLY_HASHTAG_MENTION
+        token.gsub!(Regex::ENDS_WITH_EXCITED_ONE, NOTHING)
+        token
+      end
+
+      def many_dots?(token)
+        token =~ Regex::MANY_PERIODS
       end
 
       def classic_filter!
         @tokens.map! do |token|
-          token.delete!(DOT_STRING) if @abbreviations.include?(token.chomp(DOT_STRING))
-          token.sub!(REGEX_APOSTROPHE_S, EMPTY_STRING)
+          token.delete!(DOT) if @abbreviations.include?(token.chomp(DOT))
+          token.sub!(Regex::ENDS_WITH_APOSTROPHE_AND_S, NOTHING)
           token
         end
       end
@@ -222,26 +184,26 @@ module PragmaticTokenizer
       def process_numbers!
         case @numbers
         when :semi
-          @tokens.delete_if { |t| t =~ REGEXP_NUMBER_ONLY }
+          @tokens.delete_if { |token| token =~ Regex::ONLY_DECIMALS }
         when :none
-          @tokens.delete_if { |t| t =~ REGEXP_NUMBER || PragmaticTokenizer::Languages::Common::ROMAN_NUMERALS.include?(inverse_case(t)) }
+          @tokens.delete_if { |token| token =~ Regex::NO_DECIMALS_NO_NUMERALS }
         when :only
-          @tokens.delete_if { |t| t =~ REGEXP_NO_NUMBERS }
+          @tokens.delete_if { |token| token =~ Regex::NO_DECIMALS }
         end
       end
 
       def remove_short_tokens!
-        @tokens.delete_if { |t| t.length < @minimum_length }
+        @tokens.delete_if { |token| token.length < @minimum_length }
       end
 
       def process_punctuation!
         case @punctuation
         when :semi
-          @tokens.delete_if { |t| PragmaticTokenizer::Languages::Common::SEMI_PUNCTUATION.include?(t) }
+          @tokens.delete_if { |token| token =~ Regex::PUNCTUATION4 }
         when :none
-          @tokens.delete_if { |t| PragmaticTokenizer::Languages::Common::PUNCTUATION.include?(t) || t =~ REGEXP_PUNCTUATION_ONLY }
+          @tokens.delete_if { |token| token =~ Regex::ONLY_PUNCTUATION }
         when :only
-          @tokens.keep_if { |t| PragmaticTokenizer::Languages::Common::PUNCTUATION.include?(t) }
+          @tokens.keep_if   { |token| token =~ Regex::ONLY_PUNCTUATION }
         end
       end
 
@@ -252,46 +214,50 @@ module PragmaticTokenizer
       def mentions!
         case @mentions
         when :remove
-          @tokens.delete_if { |t| t =~ REGEXP_AT_SIGN_AT_START }
+          @tokens.delete_if { |token| token =~ Regex::ONLY_MENTION }
         when :keep_and_clean
-          @tokens.map! { |t| t =~ REGEXP_AT_SIGN_AT_START ? t.gsub!(REGEXP_AT_SIGN_AT_START, EMPTY_STRING) : t }
+          @tokens.map!      { |token| token =~ Regex::ONLY_MENTION ? token[1..-1] : token }
         end
       end
 
       def hashtags!
         case @hashtags
         when :remove
-          @tokens.delete_if { |t| t =~ REGEXP_HASHTAG_AT_START }
+          @tokens.delete_if { |token| token =~ Regex::ONLY_HASHTAG }
         when :keep_and_clean
-          @tokens = @tokens
-                        .flat_map { |t| t =~ REGEXP_HYPHEN_HASTAG ? t.split(REGEX_HYPHEN) : t }
-                        .map { |t| t =~ REGEXP_HASHTAG_AT_START ? t.gsub!(REGEXP_HASHTAG_AT_START, EMPTY_STRING) : t }
+          @tokens.map!      { |token| token =~ Regex::ONLY_HASHTAG ? token[1..-1] : token }
         end
       end
 
-      def remove_various!
-        @tokens.delete_if { |t| t =~ regex_various }
+      def remove_by_options!
+        @tokens.delete_if { |token| token =~ regex_by_options }
       end
 
-      def regex_various
-        @regex_various ||= begin
+      def regex_by_options
+        @regex_by_options ||= begin
           regex_array = []
-          regex_array << REGEX_EMOJI_UNIFIED if @remove_emoji
-          regex_array << REGEX_EMAIL         if @remove_emails
-          regex_array << REGEX_URL           if @remove_urls
-          regex_array << REGEX_DOMAIN        if @remove_domains
+          regex_array << Regex::RANGE_UNUSUAL_AND_EMOJI if @remove_emoji
+          regex_array << Regex::ONLY_EMAIL              if @remove_emails
+          regex_array << Regex::STARTS_WITH_HTTP        if @remove_urls
+          regex_array << Regex::ONLY_DOMAIN2            if @remove_domains
           Regexp.union(regex_array)
         end
       end
 
       def split_long_words!
-        @tokens = @tokens
-                      .flat_map { |t| t.length > @long_word_split ? t.split(REGEX_HYPHEN) : t }
-                      .flat_map { |t| t.length > @long_word_split ? t.split(REGEX_UNDERSCORE) : t }
+        @tokens = @tokens.flat_map { |token| split_long_word(token) }
+      end
+
+      def split_long_word(token)
+        return token unless @long_word_split
+        return token if token.length <= @long_word_split
+        return token if token =~ Regex::ONLY_HASHTAG_MENTION
+        return token if token =~ Regex::DOMAIN_OR_EMAIL
+        token.split(Regex::HYPHEN_OR_UNDERSCORE)
       end
 
-      def chosen_case(token)
-        @downcase ? Unicode.downcase(token) : token
+      def chosen_case(text)
+        @downcase ? Unicode.downcase(text) : text
       end
 
       def inverse_case(token)