pragmatic_segmenter 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e0545b8e2fe6446107740b5c458b96e76b6edc51
-  data.tar.gz: 746d97aba038d8f23a6701d7df08205ff48203a8
+  metadata.gz: ae3798fa47a86a8928835153af20c91d181ab2d5
+  data.tar.gz: 265670562de5e8b25aa90454919f044c910353f8
 SHA512:
-  metadata.gz: 5975faedda7f913678ea122317266722895376da90be3b4094e50d61d2eef1e531b3df890d93199393f1945a053e14646e8d2b7bc73287de9250751f332483aa
-  data.tar.gz: d48fcd09e289833f82a5e6aa5915b3faa5ecf4417874273a2b8bb7640f51454f1374736449478da8328006277b0727c16b3b8badd258791fb011dd23f351266e
+  metadata.gz: 33eea4d021662c497763950fb5815e29b59214d2c4d7056f77f081ea3edc77a0fc9c54a9467d89dc5d278174e316ced772a6f826bb477c711239eb2b0d0b1722
+  data.tar.gz: 73fb101b5a2c6a3d2f57bdede1d37a70286519dd54ca93aaa2cfc467dd5202961bd3f27b3f25c906ea67e3038542823303bbb411bb4ffd6fafd9f7145f3aa116

data/README.md

@@ -407,6 +407,12 @@ One further habit which was somewhat weakened . . . was that of combining words
 => ["One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds.", ". . . The practice was not abandoned. . . ."]
 ```
 
+52.) **No whitespace in between sentences** *Credit: Don_Patrick*
+```
+Hello world.Today is Tuesday.Mr. Smith went to the store and bought 1,000.That is a lot.
+=> ["Hello world.", "Today is Tuesday.", "Mr. Smith went to the store and bought 1,000.", "That is a lot."]
+```
+
 ####Golden Rules (German)
 
 1.) **Quotation at end of sentence**

data/lib/pragmatic_segmenter/abbreviation_replacer.rb

@@ -80,7 +80,7 @@ module PragmaticSegmenter
     end
 
     def abbreviations
-      PragmaticSegmenter::Abbreviation.new
+      @abbr ||= PragmaticSegmenter::Abbreviation.new
     end
 
     def replace_abbreviation_as_sentence_boundary(txt)

data/lib/pragmatic_segmenter/cleaner.rb

@@ -17,6 +17,14 @@ module PragmaticSegmenter
   # xhtml, inline formatting, etc.
   class Cleaner
     include Rules
+    URL_EMAIL_KEYWORDS = ['@', 'http', '.com', 'net', 'www', '//']
+
+    # Rubular: http://rubular.com/r/6dt98uI76u
+    NO_SPACE_BETWEEN_SENTENCES_REGEX = /(?<=[a-z])\.(?=[A-Z])/
+
+    # Rubular: http://rubular.com/r/l6KN6rH5XE
+    NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX = /(?<=\d)\.(?=[A-Z])/
+
     # Rubular: http://rubular.com/r/V57WnM9Zut
     NewLineInMiddleOfWordRule = Rule.new(/\n(?=[a-zA-Z]{1,2}\n)/, '')
 
@@ -53,9 +61,17 @@ module PragmaticSegmenter
     # Rubular: http://rubular.com/r/IQ4TPfsbd8
     ConsecutiveForwardSlashRule = Rule.new(/\/{3}/, '')
 
+    # Rubular: http://rubular.com/r/6dt98uI76u
+    NoSpaceBetweenSentencesRule = Rule.new(NO_SPACE_BETWEEN_SENTENCES_REGEX, '. ')
+
+    # Rubular: http://rubular.com/r/l6KN6rH5XE
+    NoSpaceBetweenSentencesDigitRule = Rule.new(NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX, '. ')
+
     EscapedCarriageReturnRule = Rule.new(/\\r/, "\r")
+    TypoEscapedCarriageReturnRule = Rule.new(/\\\ r/, "\r")
 
     EscapedNewLineRule = Rule.new(/\\n/, "\n")
+    TypoEscapedNewLineRule = Rule.new(/\\\ n/, "\n")
 
     ReplaceNewlineWithCarriageReturnRule = Rule.new(/\n/, "\r")
 
@@ -92,11 +108,36 @@ module PragmaticSegmenter
       @clean_text.apply(InlineFormattingRule)
       clean_quotations(@clean_text)
       clean_table_of_contents(@clean_text)
+      check_for_no_space_in_between_sentences(@clean_text)
       clean_consecutive_characters(@clean_text)
     end
 
     private
 
+    def check_for_no_space_in_between_sentences(txt)
+      words = txt.split(' ')
+      words.each do |word|
+        search_for_connected_sentences(word, txt, NO_SPACE_BETWEEN_SENTENCES_REGEX, NoSpaceBetweenSentencesRule)
+        search_for_connected_sentences(word, txt, NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX, NoSpaceBetweenSentencesDigitRule)
+      end
+      txt
+    end
+
+    def search_for_connected_sentences(word, txt, regex, rule)
+      if word =~ regex
+        unless URL_EMAIL_KEYWORDS.any? { |web| word =~ /#{web}/ }
+          unless abbreviations.any? { |abbr| word =~ /#{abbr}/i }
+            new_word = word.dup.apply(rule)
+            txt.gsub!(/#{Regexp.escape(word)}/, new_word)
+          end
+        end
+      end
+    end
+
+    def abbreviations
+      @abbr ||= PragmaticSegmenter::Abbreviation.new.all
+    end
+
     def remove_all_newlines(txt)
       clean_text = remove_newline_in_middle_of_sentence(txt)
       remove_newline_in_middle_of_word(clean_text)
@@ -118,7 +159,9 @@ module PragmaticSegmenter
 
     def replace_escaped_newlines(txt)
       txt.apply(EscapedNewLineRule).
-          apply(EscapedCarriageReturnRule)
+          apply(EscapedCarriageReturnRule).
+          apply(TypoEscapedNewLineRule).
+          apply(TypoEscapedCarriageReturnRule)
     end
 
     def replace_double_newlines(txt)

data/lib/pragmatic_segmenter/languages/deutsch.rb

@@ -18,6 +18,11 @@ module PragmaticSegmenter
       end
 
       class Cleaner < PragmaticSegmenter::Cleaner
+        private
+
+        def abbreviations
+          PragmaticSegmenter::Languages::Deutsch::Abbreviation.new.all
+        end
       end
 
       class Number < PragmaticSegmenter::Number
@@ -58,7 +63,7 @@ module PragmaticSegmenter
       end
 
       class Abbreviation < PragmaticSegmenter::Abbreviation
-        ABBREVIATIONS = ['Ä', 'ä', 'adj', 'adm', 'adv', 'art', 'asst', 'b.a', 'b.s', 'bart', 'bldg', 'brig', 'bros', 'bse', 'buchst', 'bzgl', 'bzw', 'c.-à-d', 'ca', 'capt', 'chr', 'cmdr', 'co', 'col', 'comdr', 'con', 'corp', 'cpl', 'd.h', 'd.j', 'dergl', 'dgl', 'dkr', 'dr ', 'ens', 'etc', 'ev ', 'evtl', 'ff', 'g.g.a', 'g.u', 'gen', 'ggf', 'gov', 'hon', 'hosp', 'i.f', 'i.h.v', 'ii', 'iii', 'insp', 'iv', 'ix', 'jun', 'k.o', 'kath ', 'lfd', 'lt', 'ltd', 'm.e', 'maj', 'med', 'messrs', 'mio', 'mlle', 'mm', 'mme', 'mr', 'mrd', 'mrs', 'ms', 'msgr', 'mwst', 'no', 'nos', 'nr', 'o.ä', 'op', 'ord', 'pfc', 'ph', 'pp', 'prof', 'pvt', 'rep', 'reps', 'res', 'rev', 'rt', 's.p.a', 'sa', 'sen', 'sens', 'sfc', 'sgt', 'sog', 'sogen', 'spp', 'sr', 'st', 'std', 'str  ', 'supt', 'surg', 'u.a  ', 'u.e', 'u.s.w', 'u.u', 'u.ä', 'usf', 'usw', 'v', 'vgl', 'vi', 'vii', 'viii', 'vs', 'x', 'xi', 'xii', 'xiii', 'xiv', 'xix', 'xv', 'xvi', 'xvii', 'xviii', 'xx', 'z.b ', 'z.t ', 'z.z', 'z.zt', 'zt', 'zzt']
+        ABBREVIATIONS = ['Ä', 'ä', 'adj', 'adm', 'adv', 'art', 'asst', 'b.a', 'b.s', 'bart', 'bldg', 'brig', 'bros', 'bse', 'buchst', 'bzgl', 'bzw', 'c.-à-d', 'ca', 'capt', 'chr', 'cmdr', 'co', 'col', 'comdr', 'con', 'corp', 'cpl', 'd.h', 'd.j', 'dergl', 'dgl', 'dkr', 'dr ', 'ens', 'etc', 'ev ', 'evtl', 'ff', 'g.g.a', 'g.u', 'gen', 'ggf', 'gov', 'hon', 'hosp', 'i.f', 'i.h.v', 'ii', 'iii', 'insp', 'iv', 'ix', 'jun', 'k.o', 'kath ', 'lfd', 'lt', 'ltd', 'm.e', 'maj', 'med', 'messrs', 'mio', 'mlle', 'mm', 'mme', 'mr', 'mrd', 'mrs', 'ms', 'msgr', 'mwst', 'no', 'nos', 'nr', 'o.ä', 'op', 'ord', 'pfc', 'ph', 'pp', 'prof', 'pvt', 'rep', 'reps', 'res', 'rev', 'rt', 's.p.a', 'sa', 'sen', 'sens', 'sfc', 'sgt', 'sog', 'sogen', 'spp', 'sr', 'st', 'std', 'str  ', 'supt', 'surg', 'u.a  ', 'u.e', 'u.s.w', 'u.u', 'u.ä', 'usf', 'usw', 'v', 'vgl', 'vi', 'vii', 'viii', 'vs', 'x', 'xi', 'xii', 'xiii', 'xiv', 'xix', 'xv', 'xvi', 'xvii', 'xviii', 'xx', 'z.b', 'z.t', 'z.z', 'z.zt', 'zt', 'zzt']
         NUMBER_ABBREVIATIONS = ['art', 'ca', 'no', 'nos', 'nr', 'pp']
 
         def all

data/lib/pragmatic_segmenter/languages/english.rb

@@ -15,13 +15,17 @@ module PragmaticSegmenter
         def clean_quotations(txt)
           txt.gsub(/`/, "'")
         end
+
+        def abbreviations
+          []
+        end
       end
 
       class AbbreviationReplacer  < PragmaticSegmenter::AbbreviationReplacer
         private
 
         def abbreviations
-          PragmaticSegmenter::Languages::English::Abbreviation.new
+          PragmaticSegmenter::Abbreviation.new
         end
       end
     end

data/lib/pragmatic_segmenter/languages/french.rb

@@ -5,6 +5,11 @@ module PragmaticSegmenter
       end
 
       class Cleaner < PragmaticSegmenter::Cleaner
+        private
+
+        def abbreviations
+          PragmaticSegmenter::Languages::French::Abbreviation.new.all
+        end
       end
 
       class Abbreviation < PragmaticSegmenter::Abbreviation

data/lib/pragmatic_segmenter/languages/italian.rb

@@ -10,6 +10,11 @@ module PragmaticSegmenter
       end
 
       class Cleaner < PragmaticSegmenter::Cleaner
+        private
+
+        def abbreviations
+          PragmaticSegmenter::Languages::Italian::Abbreviation.new.all
+        end
       end
 
       class Abbreviation < PragmaticSegmenter::Abbreviation

data/lib/pragmatic_segmenter/languages/russian.rb

@@ -10,6 +10,11 @@ module PragmaticSegmenter
       end
 
       class Cleaner < PragmaticSegmenter::Cleaner
+        private
+
+        def abbreviations
+          PragmaticSegmenter::Languages::Russian::Abbreviation.new.all
+        end
       end
 
       class Abbreviation < PragmaticSegmenter::Abbreviation

data/lib/pragmatic_segmenter/languages/spanish.rb

@@ -10,6 +10,11 @@ module PragmaticSegmenter
       end
 
       class Cleaner < PragmaticSegmenter::Cleaner
+        private
+
+        def abbreviations
+          PragmaticSegmenter::Languages::Spanish::Abbreviation.new.all
+        end
       end
 
       class Abbreviation < PragmaticSegmenter::Abbreviation

data/lib/pragmatic_segmenter/process.rb

@@ -29,7 +29,8 @@ module PragmaticSegmenter
       reformatted_text = PragmaticSegmenter::List.new(text: text).add_line_break
       reformatted_text = replace_abbreviations(reformatted_text)
       reformatted_text = replace_numbers(reformatted_text)
-      reformatted_text = reformatted_text.apply(GeoLocationRule)
+      reformatted_text.apply(AbbreviationsWithMultiplePeriodsAndEmailRule)
+      reformatted_text.apply(GeoLocationRule)
       split_into_segments(reformatted_text)
     end
 
@@ -37,7 +38,7 @@ module PragmaticSegmenter
 
     def split_into_segments(txt)
       txt.split("\r")
-         .map! { |segment| segment.apply(SingleNewLineRule, EllipsisRules::All, EmailRule) }
+         .map! { |segment| segment.apply(SingleNewLineRule, EllipsisRules::All) }
          .map { |segment| check_for_punctuation(segment) }.flatten
          .map! { |segment| segment.apply(SubSymbolsRules::All) }
          .map { |segment| post_process_segments(segment) }

data/lib/pragmatic_segmenter/rules.rb

@@ -1,7 +1,7 @@
 module PragmaticSegmenter
   module Rules
     # Rubular: http://rubular.com/r/EUbZCNfgei
-    EmailRule = Rule.new(/(\w)(\.)(\w)/, '\1∮\3')
+    AbbreviationsWithMultiplePeriodsAndEmailRule = Rule.new(/(\w)(\.)(\w)/, '\1∮\3')
 
     # Rubular: http://rubular.com/r/G2opjedIm9
     GeoLocationRule = Rule.new(/(?<=[a-zA-z]°)\.(?=\s*\d+)/, '∯')

data/lib/pragmatic_segmenter/segmenter.rb

@@ -32,7 +32,6 @@ module PragmaticSegmenter
       @language = args[:language] || 'en'
       @doc_type = args[:doc_type]
       @text = text.dup
-
       unless args[:clean].eql?(false)
         @text = cleaner_class.new(text: @text, doc_type: args[:doc_type]).clean
       end
@@ -40,7 +39,6 @@ module PragmaticSegmenter
 
     def segment
       return [] unless text
-
       process_class.new(text: text, doc_type: doc_type).process
     end
   end

data/lib/pragmatic_segmenter/version.rb

@@ -1,3 +1,3 @@
 module PragmaticSegmenter
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/spec/pragmatic_segmenter_spec.rb

@@ -258,6 +258,11 @@ RSpec.describe PragmaticSegmenter::Segmenter do
       ps = PragmaticSegmenter::Segmenter.new(text: "One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds. . . . The practice was not abandoned. . . .", language: "en")
       expect(ps.segment).to eq(["One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds.", ". . . The practice was not abandoned. . . ."])
     end
+
+    it "No whitespace in between sentences #052" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Hello world.Today is Tuesday.Mr. Smith went to the store and bought 1,000.That is a lot.", language: "en")
+      expect(ps.segment).to eq(["Hello world.", "Today is Tuesday.", "Mr. Smith went to the store and bought 1,000.", "That is a lot."])
+    end
   end
 
   context "Golden Rules (languages other than English)" do
@@ -873,6 +878,11 @@ RSpec.describe PragmaticSegmenter::Segmenter do
         ps = PragmaticSegmenter::Segmenter.new(text: 'Hello World. \r\n Hello.', language: 'en')
         expect(ps.segment).to eq(["Hello World.", "Hello."])
       end
+
+      it 'correctly segments text #082' do
+        ps = PragmaticSegmenter::Segmenter.new(text: 'Hello World. \ r \ nHello.', language: 'en')
+        expect(ps.segment).to eq(["Hello World.", "Hello."])
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-10 00:00:00.000000000 Z
+date: 2015-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler