pragmatic_segmenter 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d58b0da895c9249efd632eca9c21ba386ab0ba3
-  data.tar.gz: 3e9b46ad1c9bbc1704fe2a6b2c4ce899c73267fa
+  metadata.gz: e0545b8e2fe6446107740b5c458b96e76b6edc51
+  data.tar.gz: 746d97aba038d8f23a6701d7df08205ff48203a8
 SHA512:
-  metadata.gz: 14b408a8e527d35d6da36082fa17240c1c9c6b50f66747fda05e52ed859ca0d1c3b6f5a8f57ef8fd0827b5134d960beed9536509738c94db31dbcd12b3a03fcd
-  data.tar.gz: 6c67d78cf0777504c4c21e6cf9a168897b4c8c43bf3e3e84a6161f81bcbb8355a5794d35a9924196fafed4a0d682ff74063548a676d454e1a198ad3ea9959a83
+  metadata.gz: 5975faedda7f913678ea122317266722895376da90be3b4094e50d61d2eef1e531b3df890d93199393f1945a053e14646e8d2b7bc73287de9250751f332483aa
+  data.tar.gz: d48fcd09e289833f82a5e6aa5915b3faa5ecf4417874273a2b8bb7640f51454f1374736449478da8328006277b0727c16b3b8badd258791fb011dd23f351266e

data/lib/pragmatic_segmenter/cleaner.rb

@@ -50,9 +50,13 @@ module PragmaticSegmenter
     # Rubular: http://rubular.com/r/DwNSuZrNtk
     ConsecutivePeriodsRule = Rule.new(/\.{5,}/, ' ')
 
-    # http://rubular.com/r/IQ4TPfsbd8
+    # Rubular: http://rubular.com/r/IQ4TPfsbd8
     ConsecutiveForwardSlashRule = Rule.new(/\/{3}/, '')
 
+    EscapedCarriageReturnRule = Rule.new(/\\r/, "\r")
+
+    EscapedNewLineRule = Rule.new(/\\n/, "\n")
+
     ReplaceNewlineWithCarriageReturnRule = Rule.new(/\n/, "\r")
 
     QuotationsFirstRule = Rule.new(/''/, '"')
@@ -81,12 +85,14 @@ module PragmaticSegmenter
     def clean
       return unless text
       @clean_text = remove_all_newlines(text)
-      @clean_text = replace_double_newlines(@clean_text)
-      @clean_text = replace_newlines(@clean_text)
-      @clean_text = @clean_text.apply(HtmlRules::All)
-      @clean_text = @clean_text.apply(InlineFormattingRule)
-      @clean_text = clean_quotations(@clean_text)
-      @clean_text = clean_table_of_contents(@clean_text)
+      replace_double_newlines(@clean_text)
+      replace_newlines(@clean_text)
+      replace_escaped_newlines(@clean_text)
+      @clean_text.apply(HtmlRules::All)
+      @clean_text.apply(InlineFormattingRule)
+      clean_quotations(@clean_text)
+      clean_table_of_contents(@clean_text)
+      clean_consecutive_characters(@clean_text)
     end
 
     private
@@ -110,6 +116,11 @@ module PragmaticSegmenter
       txt.apply(NewLineInMiddleOfWordRule)
     end
 
+    def replace_escaped_newlines(txt)
+      txt.apply(EscapedNewLineRule).
+          apply(EscapedCarriageReturnRule)
+    end
+
     def replace_double_newlines(txt)
       txt.apply(DoubleNewLineWithSpaceRule).
           apply(DoubleNewLineRule)
@@ -117,13 +128,11 @@ module PragmaticSegmenter
 
     def replace_newlines(txt)
       if doc_type.eql?('pdf')
-        txt = remove_pdf_line_breaks(txt)
+        remove_pdf_line_breaks(txt)
       else
-        txt =
-          txt.apply(NewLineFollowedByPeriodRule).
-              apply(ReplaceNewlineWithCarriageReturnRule)
+        txt.apply(NewLineFollowedByPeriodRule).
+            apply(ReplaceNewlineWithCarriageReturnRule)
       end
-      txt
     end
 
     def remove_pdf_line_breaks(txt)
@@ -142,5 +151,10 @@ module PragmaticSegmenter
           apply(ConsecutivePeriodsRule).
           apply(ConsecutiveForwardSlashRule)
     end
+
+    def clean_consecutive_characters(txt)
+      txt.apply(ConsecutivePeriodsRule).
+          apply(ConsecutiveForwardSlashRule)
+    end
   end
 end

data/lib/pragmatic_segmenter/ellipsis.rb

@@ -30,7 +30,6 @@ module PragmaticSegmenter
         ThreeConsecutiveRule,
         OtherThreePeriodRule
       ]
-
     end
   end
 end

data/lib/pragmatic_segmenter/language_support.rb

@@ -25,6 +25,5 @@ module PragmaticSegmenter
     def cleaner_class
       Object.const_get("PragmaticSegmenter::Languages::#{LANGUAGE_CODES[language] || 'Common'}::Cleaner")
     end
-
   end
 end

data/lib/pragmatic_segmenter/languages/deutsch.rb

@@ -29,7 +29,7 @@ module PragmaticSegmenter
 
         def replace
           super
-          @formatted_text.apply(NumberPeriodSpaceRule).apply(NegativeNumberPeriodSpaceRule)
+          @text.apply(NumberPeriodSpaceRule).apply(NegativeNumberPeriodSpaceRule)
         end
       end
 

data/lib/pragmatic_segmenter/list.rb

@@ -124,14 +124,12 @@ module PragmaticSegmenter
       txt.gsub!(ALPHABETICAL_LIST_LETTERS_AND_PERIODS_REGEX).with_index do |m|
         a.eql?(m.chomp('.')) ? "\r#{Regexp.escape(a.to_s)}∯" : "#{m}"
       end
-      txt
     end
 
     def replace_alphabet_list_parens(a, txt)
       txt.gsub!(EXTRACT_ALPHABETICAL_LIST_LETTERS_REGEX).with_index do |m|
         a.eql?(m) ? "\r#{Regexp.escape(a.to_s)}" : "#{m}"
       end
-      txt
     end
 
     def replace_correct_alphabet_list(a, txt, parens)

data/lib/pragmatic_segmenter/number.rb

@@ -25,11 +25,11 @@ module PragmaticSegmenter
     end
 
     def replace
-      @formatted_text = @text.apply(PeriodBeforeNumberRule).
-          apply(NumberAfterPeriodBeforeLetterRule).
-          apply(NewLineNumberPeriodSpaceLetterRule).
-          apply(StartLineNumberPeriodRule).
-          apply(StartLineTwoDigitNumberPeriodRule)
+      @text.apply(PeriodBeforeNumberRule).
+            apply(NumberAfterPeriodBeforeLetterRule).
+            apply(NewLineNumberPeriodSpaceLetterRule).
+            apply(StartLineNumberPeriodRule).
+            apply(StartLineTwoDigitNumberPeriodRule)
     end
   end
 end

data/lib/pragmatic_segmenter/process.rb

@@ -30,57 +30,53 @@ module PragmaticSegmenter
       reformatted_text = replace_abbreviations(reformatted_text)
       reformatted_text = replace_numbers(reformatted_text)
       reformatted_text = reformatted_text.apply(GeoLocationRule)
-      split_lines(reformatted_text)
+      split_into_segments(reformatted_text)
     end
 
     private
 
-    def split_lines(txt)
-      segments = txt.split("\r")
+    def split_into_segments(txt)
+      txt.split("\r")
+         .map! { |segment| segment.apply(SingleNewLineRule, EllipsisRules::All, EmailRule) }
+         .map { |segment| check_for_punctuation(segment) }.flatten
+         .map! { |segment| segment.apply(SubSymbolsRules::All) }
+         .map { |segment| post_process_segments(segment) }
+         .flatten.compact.delete_if(&:empty?)
+    end
 
-      segments.map! do |line|
-        line.apply(SingleNewLineRule, EllipsisRules::All, EmailRule)
+    def post_process_segments(txt)
+      return if consecutive_underscore?(txt) || txt.length < 2
+      txt.apply(ReinsertEllipsisRules::All).apply(ExtraWhiteSpaceRule)
+      if txt =~ QUOTATION_AT_END_OF_SENTENCE_REGEX
+        txt.split(SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX)
+      else
+        txt.tr("\n", '').strip
       end
+    end
 
-      segments = segments.map { |line| analyze_lines(line) }.flatten
-
-      segments.map! {|segment| sub_symbols(segment) }
-
-      sentence_array = []
-      segments.each_with_index do |line|
-        next if line.gsub(/_{3,}/, '').length.eql?(0) || line.length < 2
-        line = reinsert_ellipsis(line)
-        line = line.apply(ExtraWhiteSpaceRule)
-        if line =~ QUOTATION_AT_END_OF_SENTENCE_REGEX
-          subline = line.split(SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX)
-          subline.each do |s|
-            sentence_array << s
-          end
-        else
-          sentence_array << line.tr("\n", '').strip
-        end
-      end
-      sentence_array.reject(&:empty?)
+    def consecutive_underscore?(txt)
+      # Rubular: http://rubular.com/r/fTF2Ff3WBL
+      txt.gsub(/_{3,}/, '').length.eql?(0)
     end
 
-    def analyze_lines(line)
-      if punctuation_array.any? { |p| line.include?(p) }
-        process_text(line)
+    def check_for_punctuation(txt)
+      if punctuation_array.any? { |p| txt.include?(p) }
+        process_text(txt)
       else
-        line
+        txt
       end
     end
 
-    def process_text(line)
-      line << 'ȸ' unless punctuation_array.any? { |p| line[-1].include?(p) }
-      PragmaticSegmenter::ExclamationWords.apply_rules(line)
-      between_punctutation(line)
-      line = line.apply(
+    def process_text(txt)
+      txt << 'ȸ' unless punctuation_array.any? { |p| txt[-1].include?(p) }
+      PragmaticSegmenter::ExclamationWords.apply_rules(txt)
+      between_punctutation(txt)
+      txt = txt.apply(
         DoublePuctationRules::All,
         QuestionMarkInQuotationRule,
         ExclamationPointRules::All
       )
-      sentence_boundary_punctuation(line)
+      sentence_boundary_punctuation(txt)
     end
 
     def replace_numbers(txt)
@@ -102,17 +98,5 @@ module PragmaticSegmenter
     def sentence_boundary_punctuation(txt)
       PragmaticSegmenter::SentenceBoundaryPunctuation.new(text: txt).split
     end
-
-    def sub_symbols(txt)
-      txt.gsub(/∯/, '.').gsub(/♬/, '،').gsub(/♭/, ':').gsub(/ᓰ/, '。').gsub(/ᓱ/, '．')
-        .gsub(/ᓳ/, '！').gsub(/ᓴ/, '!').gsub(/ᓷ/, '?').gsub(/ᓸ/, '？').gsub(/☉/, '?!')
-        .gsub(/☈/, '!?').gsub(/☇/, '??').gsub(/☄/, '!!').delete('ȸ').gsub(/ȹ/, "\n")
-    end
-
-    def reinsert_ellipsis(line)
-      line.gsub(/ƪ/, '...').gsub(/♟/, ' . . . ')
-        .gsub(/♝/, '. . . .').gsub(/☏/, '..')
-        .gsub(/∮/, '.')
-    end
   end
 end

data/lib/pragmatic_segmenter/punctuation_replacer.rb

@@ -4,6 +4,7 @@ module PragmaticSegmenter
   # This class replaces punctuation that is typically a sentence boundary
   # but in this case is not a sentence boundary.
   class PunctuationReplacer
+    include Rules
     attr_reader :matches_array, :text
     def initialize(text:, matches_array:)
       @text = text
@@ -18,45 +19,24 @@ module PragmaticSegmenter
 
     def replace_punctuation(array, txt)
       return if !array || array.empty?
-      txt.gsub!('(', '\\(')
-      txt.gsub!(')', '\\)')
-      txt.gsub!(']', '\\]')
-      txt.gsub!('[', '\\[')
-      txt.gsub!('-', '\\-')
+      txt.apply(EscapeRegexReservedCharacters::All)
       array.each do |a|
-        a.gsub!('(', '\\(')
-        a.gsub!(')', '\\)')
-        a.gsub!(']', '\\]')
-        a.gsub!('[', '\\[')
-        a.gsub!('-', '\\-')
-
-        sub = a.gsub('.', '∯')
-        txt.gsub!(/#{Regexp.escape(a)}/, "#{sub}")
-
-        sub_1 = sub.gsub('。', 'ᓰ')
-        txt.gsub!(/#{Regexp.escape(sub)}/, "#{sub_1}")
-
-        sub_2 = sub_1.gsub('．', 'ᓱ')
-        txt.gsub!(/#{Regexp.escape(sub_1)}/, "#{sub_2}")
-
-        sub_3 = sub_2.gsub('！', 'ᓳ')
-        txt.gsub!(/#{Regexp.escape(sub_2)}/, "#{sub_3}")
-
-        sub_4 = sub_3.gsub('!', 'ᓴ')
-        txt.gsub!(/#{Regexp.escape(sub_3)}/, "#{sub_4}")
-
-        sub_5 = sub_4.gsub('?', 'ᓷ')
-        txt.gsub!(/#{Regexp.escape(sub_4)}/, "#{sub_5}")
-
-        sub_6 = sub_5.gsub('？', 'ᓸ')
-        txt.gsub!(/#{Regexp.escape(sub_5)}/, "#{sub_6}")
+        a.apply(EscapeRegexReservedCharacters::All)
+        sub = sub_characters(txt, a, '.', '∯')
+        sub_1 = sub_characters(txt, sub, '。', '&ᓰ&')
+        sub_2 = sub_characters(txt, sub_1, '．', '&ᓱ&')
+        sub_3 = sub_characters(txt, sub_2, '！', '&ᓳ&')
+        sub_4 = sub_characters(txt, sub_3, '!', '&ᓴ&')
+        sub_5 = sub_characters(txt, sub_4, '?', '&ᓷ&')
+        sub_6 = sub_characters(txt, sub_5, '？', '&ᓸ&')
       end
-      txt.gsub!('\\(', '(')
-      txt.gsub!('\\)', ')')
-      txt.gsub!('\\[', '[')
-      txt.gsub!('\\]', ']')
-      txt.gsub!('\\-', '-')
-      txt
+      txt.apply(SubEscapedRegexReservedCharacters::All)
+    end
+
+    def sub_characters(txt, string, char_a, char_b)
+      sub = string.gsub(char_a, char_b)
+      txt.gsub!(/#{Regexp.escape(string)}/, "#{sub}")
+      sub
     end
   end
 end

data/lib/pragmatic_segmenter/rules.rb

@@ -11,17 +11,17 @@ module PragmaticSegmenter
     ExtraWhiteSpaceRule = Rule.new(/\s{3,}/, ' ')
 
     # Rubular: http://rubular.com/r/aXPUGm6fQh
-    QuestionMarkInQuotationRule = Rule.new(/\?(?=(\'|\"))/, 'ᓷ')
+    QuestionMarkInQuotationRule = Rule.new(/\?(?=(\'|\"))/, '&ᓷ&')
 
     module ExclamationPointRules
       # Rubular: http://rubular.com/r/XS1XXFRfM2
-      InQuotationRule = Rule.new(/\!(?=(\'|\"))/, 'ᓴ')
+      InQuotationRule = Rule.new(/\!(?=(\'|\"))/, '&ᓴ&')
 
       # Rubular: http://rubular.com/r/sl57YI8LkA
-      BeforeCommaMidSentenceRule = Rule.new(/\!(?=\,\s[a-z])/, 'ᓴ')
+      BeforeCommaMidSentenceRule = Rule.new(/\!(?=\,\s[a-z])/, '&ᓴ&')
 
       # Rubular: http://rubular.com/r/f9zTjmkIPb
-      MidSentenceRule = Rule.new(/\!(?=\s[a-z])/, 'ᓴ')
+      MidSentenceRule = Rule.new(/\!(?=\s[a-z])/, '&ᓴ&')
 
       All = [ InQuotationRule, BeforeCommaMidSentenceRule, MidSentenceRule ]
     end
@@ -34,5 +34,66 @@ module PragmaticSegmenter
 
       All = [ FirstRule, SecondRule, ThirdRule, ForthRule ]
     end
+
+    module ReinsertEllipsisRules
+      ThreeConsecutivePeriod = Rule.new(/ƪ/, '...')
+      ThreeSpacePeriod = Rule.new(/♟/, ' . . . ')
+      FourSpacePeriod = Rule.new(/♝/, '. . . .')
+      TwoConsecutivePeriod = Rule.new(/☏/, '..')
+      OnePeriod = Rule.new(/∮/, '.')
+
+      All = [ ThreeConsecutivePeriod, ThreeSpacePeriod,
+              FourSpacePeriod, TwoConsecutivePeriod,
+              OnePeriod ]
+    end
+
+    module SubSymbolsRules
+      Period = Rule.new(/∯/, '.')
+      ArabicComma = Rule.new(/♬/, '،')
+      SemiColon = Rule.new(/♭/, ':')
+      FullWidthPeriod = Rule.new(/&ᓰ&/, '。')
+      SpecialPeriod = Rule.new(/&ᓱ&/, '．')
+      FullWidthExclamation = Rule.new(/&ᓳ&/, '！')
+      ExclamationPoint = Rule.new(/&ᓴ&/, '!')
+      QuestionMark = Rule.new(/&ᓷ&/, '?')
+      FullWidthQuestionMark = Rule.new(/&ᓸ&/, '？')
+      MixedDoubleQE = Rule.new(/☉/, '?!')
+      MixedDoubleQQ = Rule.new(/☇/, '??')
+      MixedDoubleEQ = Rule.new(/☈/, '!?')
+      MixedDoubleEE = Rule.new(/☄/, '!!')
+      TemporaryEndingPunctutation = Rule.new('ȸ', '')
+      Newline = Rule.new(/ȹ/, "\n")
+
+      All = [ Period, ArabicComma,
+              SemiColon, FullWidthPeriod,
+              SpecialPeriod, FullWidthExclamation,
+              ExclamationPoint, QuestionMark,
+              FullWidthQuestionMark, MixedDoubleQE,
+              MixedDoubleQQ, MixedDoubleEQ,
+              MixedDoubleEE, TemporaryEndingPunctutation,
+              Newline ]
+    end
+
+    module EscapeRegexReservedCharacters
+      LeftParen = Rule.new('(', '\\(')
+      RightParen = Rule.new(')', '\\)')
+      LeftBracket = Rule.new('[', '\\[')
+      RightBracket = Rule.new(']', '\\]')
+      Dash = Rule.new('-', '\\-')
+
+      All = [ LeftParen, RightParen,
+              LeftBracket, RightBracket, Dash ]
+    end
+
+    module SubEscapedRegexReservedCharacters
+      LeftParen = Rule.new('\\(', '(')
+      RightParen = Rule.new('\\)', ')')
+      LeftBracket = Rule.new('\\[', '[')
+      RightBracket = Rule.new('\\]', ']')
+      Dash = Rule.new('\\-', '-')
+
+      All = [ LeftParen, RightParen,
+              LeftBracket, RightBracket, Dash ]
+    end
   end
 end

data/lib/pragmatic_segmenter/version.rb

@@ -1,3 +1,3 @@
 module PragmaticSegmenter
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/spec/pragmatic_segmenter_spec.rb

@@ -868,6 +868,11 @@ RSpec.describe PragmaticSegmenter::Segmenter do
         ps = PragmaticSegmenter::Segmenter.new(text: "////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Header starts here\r////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////", language: 'en')
         expect(ps.segment).to eq(["Header starts here"])
       end
+
+      it 'correctly segments text #082' do
+        ps = PragmaticSegmenter::Segmenter.new(text: 'Hello World. \r\n Hello.', language: 'en')
+        expect(ps.segment).to eq(["Hello World.", "Hello."])
+      end
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Kevin S. Dias