pragmatic_tokenizer 1.4.0 → 1.5.0

data/lib/pragmatic_tokenizer/pre_processor.rb

@@ -1,148 +1,178 @@
 module PragmaticTokenizer
-  class PreProcessor
-
-    def initialize(language: Languages::Common)
-      @language = language
-    end
-
-    def pre_process(text:)
-      shift_comma(text)
-      shift_multiple_dash(text)
-      shift_upsidedown_question_mark(text)
-      shift_upsidedown_exclamation(text)
-      shift_exclamation(text)
-      shift_ellipse(text)
-      shift_no_space_mention(text)
-      shift_not_equals(text)
-      shift_special_quotes(text)
-      shift_colon(text)
-      shift_bracket(text)
-      shift_semicolon(text)
-      shift_percent(text)
-      shift_caret(text)
-      shift_hashtag(text)
-      shift_ampersand(text)
-      shift_vertical_bar(text)
-      convert_dbl_quotes(text)
-      convert_sgl_quotes(text)
-      convert_apostrophe_s(text)
-      shift_beginning_hyphen(text)
-      shift_ending_hyphen(text)
-      text.squeeze(' ')
+  module PreProcessor
+
+    def pre_process(language: Languages::Common)
+      shift_comma!
+      shift_multiple_dash!
+      shift_inverted_question_mark!
+      shift_inverted_exclamation!
+      shift_exclamation!
+      shift_ellipse_three_dots!
+      shift_ellipse_two_dots!
+      shift_horizontal_ellipsis!
+      shift_no_space_mention!
+      shift_not_equals!
+      shift_special_quotes!
+      shift_colon!
+      shift_bracket!
+      shift_semicolon!
+      shift_percent!
+      shift_caret!
+      shift_hashtag!
+      shift_ampersand!
+      shift_vertical_bar!
+      convert_dbl_quotes!
+      convert_sgl_quotes!(language)
+      convert_apostrophe_s!
+      shift_beginning_hyphen!
+      shift_ending_hyphen!
+      squeeze(' '.freeze)
     end
 
     private
 
-    def shift_comma(text)
       # Shift commas off everything but numbers
-      text.gsub!(/,(?!\d)/o, ' , ') || text
-      text.gsub!(/(?<=\D),(?=\S+)/, ' , ') || text
-    end
+      def shift_comma!
+        gsub!(/,(?!\d)/o, ' , '.freeze)
+        gsub!(/(?<=\D),(?=\S+)/, ' , '.freeze)
+      end
 
-    def shift_multiple_dash(text)
-      text.gsub!(/--+/o, ' - ') || text
-    end
+      def shift_multiple_dash!
+        gsub!(/--+/o, ' - '.freeze)
+      end
 
-    def shift_upsidedown_question_mark(text)
-      text.gsub!(/¿/, ' ¿ ') || text
-    end
+      def shift_inverted_question_mark!
+        gsub!(/¿/, ' ¿ '.freeze)
+      end
 
-    def shift_upsidedown_exclamation(text)
-      text.gsub!(/¡/, ' ¡ ') || text
-    end
+      def shift_inverted_exclamation!
+        gsub!(/¡/, ' ¡ '.freeze)
+      end
 
-    def shift_exclamation(text)
-      text.gsub!(/(?<=[a-zA-z])!(?=[a-zA-z])/, ' ! ') || text
-    end
+      def shift_exclamation!
+        gsub!(/(?<=[a-zA-z])!(?=[a-zA-z])/, ' ! '.freeze)
+      end
 
-    def shift_ellipse(text)
-      text.gsub!(/(\.\.\.+)/o) { ' ' + $1 + ' ' } || text
-      text.gsub!(/(\.\.+)/o) { ' ' + $1 + ' ' } || text
-      text.gsub!(/(…+)/o) { ' ' + $1 + ' ' } || text
-    end
+      def shift_horizontal_ellipsis!
+        gsub!(/(…+)/o) { ' '.freeze + Regexp.last_match(1) + ' '.freeze }
+      end
 
-    def shift_no_space_mention(text)
-      text.gsub!(/\.(?=(@|＠)[^\.]+(\s|\z))/, '. ') || text
-    end
+      def shift_ellipse_two_dots!
+        gsub!(/(\.\.+)/o) { ' '.freeze + Regexp.last_match(1) + ' '.freeze }
+      end
 
-    def shift_not_equals(text)
-      text.gsub!(/≠/, ' ≠ ') || text
-    end
+      def shift_ellipse_three_dots!
+        gsub!(/(\.\.\.+)/o) { ' '.freeze + Regexp.last_match(1) + ' '.freeze }
+      end
 
-    def shift_special_quotes(text)
-      text.gsub!(/«/, ' « ') || text
-      text.gsub!(/»/, ' » ') || text
-      text.gsub!(/„/, ' „ ') || text
-      text.gsub!(/“/, ' “ ') || text
-    end
+      def shift_no_space_mention!
+        gsub!(/\.(?=(@|＠)[^\.]+(\s|\z))/, '. '.freeze)
+      end
 
-    def shift_colon(text)
-      return text unless text.include?(':') &&
-        (text.partition(':').last[0] !~ /\A\d+/ ||
-        text.partition(':').first[-1] !~ /\A\d+/)
-      # Ignore web addresses
-      text.gsub!(/(?<=[http|https]):(?=\/\/)/, PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP[":"]) || text
-      text.gsub!(/:/o, ' :') || text
-      text.gsub!(/(?<=\s):(?=\#)/, ': ') || text
-    end
+      def shift_not_equals!
+        gsub!(/≠/, ' ≠ '.freeze)
+      end
 
-    def shift_bracket(text)
-      text.gsub!(/([\(\[\{\}\]\)])/o) { ' ' + $1 + ' ' } || text
-    end
+      def shift_special_quotes!
+        gsub!(/([«»„“])/, ' \1 ')
+      end
 
-    def shift_semicolon(text)
-      text.gsub!(/([;])/o) { ' ' + $1 + ' ' } || text
-    end
+      def shift_colon!
+        return unless may_shift_colon?
+        # Ignore web addresses
+        replacement = replacement_for_key(':'.freeze)
+        gsub!(%r{(?<=[(https?|ftp)]):(?=//)}, replacement)
+        gsub!(/:/o, ' :'.freeze)
+        gsub!(/(?<=\s):(?=\#)/, ': '.freeze)
+      end
 
-    def shift_percent(text)
-      text.gsub!(/(?<=\D)%(?=\d+)/, ' %') || text
-    end
+      def may_shift_colon?
+        return false unless include?(':'.freeze)
+        partitions = partition(':'.freeze)
+        partitions.last[0] !~ /\A\d+/ || partitions.first[-1] !~ /\A\d+/
+      end
 
-    def shift_caret(text)
-      text.gsub!(/\^/, ' ^ ') || text
-    end
+      def shift_bracket!
+        gsub!(/([\(\[\{\}\]\)])/o) { ' ' + Regexp.last_match(1) + ' '.freeze }
+      end
 
-    def shift_hashtag(text)
-      text.gsub!(/(?<=\S)(#|＃)(?=\S)/, ' \1\2') || text
-    end
+      def shift_semicolon!
+        gsub!(/([;])/o) { ' '.freeze + Regexp.last_match(1) + ' '.freeze }
+      end
 
-    def shift_ampersand(text)
-      text.gsub!(/\&/, ' & ') || text
-    end
+      def shift_percent!
+        gsub!(/(?<=\D)%(?=\d+)/, ' %'.freeze)
+      end
 
-    def shift_vertical_bar(text)
-      text.gsub!(/\|/, ' | ') || text
-    end
+      def shift_caret!
+        gsub!(/\^/, ' ^ '.freeze)
+      end
 
-    def convert_dbl_quotes(text)
-      # Convert left double quotes to special character
-      text.gsub!(/''(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP['"'] + ' ') || text
-      text.gsub!(/"(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP['"'] + ' ') || text
-      text.gsub!(/“(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP['“'] + ' ') || text
-      # Convert remaining quotes to special character
-      text.gsub!(/"/, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP['"'] + ' ') || text
-      text.gsub!(/''/, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP['"'] + ' ') || text
-      text.gsub!(/”/, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP['”'] + ' ') || text
-    end
+      def shift_hashtag!
+        gsub!(/(?<=\S)(#|＃)(?=\S)/, ' \1\2')
+      end
 
-    def convert_sgl_quotes(text)
-      if defined? @language::SingleQuotes
-        @language::SingleQuotes.new.handle_single_quotes(text)
-      else
-        PragmaticTokenizer::Languages::Common::SingleQuotes.new.handle_single_quotes(text)
+      def shift_ampersand!
+        gsub!(/\&/, ' & '.freeze)
       end
-    end
 
-    def convert_apostrophe_s(text)
-      text.gsub!(/\s\u{0301}(?=s(\s|\z))/, PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP['`']) || text
-    end
+      def shift_vertical_bar!
+        gsub!(/\|/, ' | '.freeze)
+      end
 
-    def shift_beginning_hyphen(text)
-      text.gsub!(/\s+-/, ' - ') || text
-    end
+      def convert_dbl_quotes!
+        replace_left_double_quotes!
+        replace_remaining_double_quotes!
+      end
+
+      def replace_left_double_quotes!
+        replace_left_quotes!("''", '"'.freeze)
+        replace_left_quotes!('"', '"'.freeze)
+        replace_left_quotes!('“', '“'.freeze)
+      end
+
+      def replace_left_quotes!(style, replacement_key)
+        replacement = replacement_for_key(replacement_key)
+        gsub!(/#{style}(?=.*\w)/o, ' '.freeze + replacement + ' '.freeze)
+      end
+
+      def replace_remaining_double_quotes!
+        replace_remaining_quotes!('"', '"'.freeze)
+        replace_remaining_quotes!("''", '"'.freeze)
+        replace_remaining_quotes!('”', '”'.freeze)
+      end
+
+      def replace_remaining_quotes!(style, replacement_key)
+        replacement = replacement_for_key(replacement_key)
+        gsub!(/#{style}/, ' '.freeze + replacement + ' '.freeze)
+      end
+
+      def convert_sgl_quotes!(language)
+        replace(if defined?(language::SingleQuotes)
+                  language::SingleQuotes.new
+                      .handle_single_quotes(self)
+                else
+                  PragmaticTokenizer::Languages::Common::SingleQuotes.new
+                      .handle_single_quotes(self)
+                end)
+      end
+
+      def convert_apostrophe_s!
+        replacement = replacement_for_key('`'.freeze)
+        gsub!(/\s\u{0301}(?=s(\s|\z))/, replacement)
+      end
+
+      def shift_beginning_hyphen!
+        gsub!(/\s+-/, ' - '.freeze)
+      end
+
+      def shift_ending_hyphen!
+        gsub!(/-\s+/, ' - '.freeze)
+      end
+
+      def replacement_for_key(replacement_key)
+        PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP[replacement_key]
+      end
 
-    def shift_ending_hyphen(text)
-      text.gsub!(/-\s+/, ' - ') || text
-    end
   end
 end