pragmatic_segmenter 0.3.14 → 0.3.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f4cb40793083044dcf653f7c5c2c64d6e1f9476
-  data.tar.gz: df80bb40afbb05d6f9d1109f0fbb412b34fd7171
+  metadata.gz: d67794ddb8ba2e0fdcdd4b0bda833ec07d7addc0
+  data.tar.gz: 04e7435b5148d865aca9062a9c72f88f8768860e
 SHA512:
-  metadata.gz: 3296133dffb5dc2a1c3084f31ecd6115c9e5ffbcb53590ee751fa1e69159dbffadc0bbb6885f7fb09f38ec53aceaf07e2da8942ba6135ee344ae34fd1fac4d83
-  data.tar.gz: 6c3230195453e40c8a4bbfcb9b864a369db7a3beebcf4f61f37b11f2f8dff454a173456a59fa298d5962560441828fbec6b3d926e4dd033785cd4b8a7a716a05
+  metadata.gz: c23512769b5ad8d8190ff84443662d41959960352b45e5c108ed8417d12953f44be37ce8117b2434bb72c411b4c905472525e1411984e5e3cd39de97d1865b93
+  data.tar.gz: 6e108d84ad98a09e5cc6bfa51881541eb1f90b7103e7c004a1cd2980a3bed0c970e850fb5df4231af326dd15b7eaf04e1a09f5c96e849e2942a3e8dbbf94638f

data/NEWS

@@ -1,4 +1,8 @@
-0.3.13 (2017-06-28):
+0.3.15 (2017-06-28):
+
+* Improvement: Handle em dashes that appear in the middle of a sentence and include a sentence ending punctuation mark
+
+0.3.14 (2017-06-28):
 
 * Improvement: Add English abbreviation Rs. to denote the Indian currency
 

data/README.md

@@ -853,6 +853,9 @@ To test the relative performance of different segmentation tools and libraries I
 **Version 0.3.14**  
 * Add English abbreviation Rs. to denote the Indian currency
 
+**Version 0.3.15** 
+* Handle em dashes that appear in the middle of a sentence and include a sentence ending punctuation mark
+
 ## Contributing
 
 If you find a text that is incorrectly segmented using this gem, please submit an issue.

data/lib/pragmatic_segmenter/between_punctuation.rb

@@ -25,6 +25,9 @@ module PragmaticSegmenter
     # Rubular: http://rubular.com/r/mXf8cW025o
     WORD_WITH_LEADING_APOSTROPHE = /(?<=\s)'(?:[^']|'[a-zA-Z])*'\S/
 
+    # Rubular: http://rubular.com/r/jTtDKfjxzr
+    BETWEEN_EM_DASHES_REGEX = /\-\-(?>[^\-\-])*\-\-/
+
     attr_reader :text
     def initialize(text:)
       @text = text
@@ -42,6 +45,7 @@ module PragmaticSegmenter
       sub_punctuation_between_square_brackets(txt)
       sub_punctuation_between_parens(txt)
       sub_punctuation_between_quotes_arrow(txt)
+      sub_punctuation_between_em_dashes(txt)
       sub_punctuation_between_quotes_slanted(txt)
     end
 
@@ -87,6 +91,13 @@ module PragmaticSegmenter
       ).replace
     end
 
+    def sub_punctuation_between_em_dashes(txt)
+      PragmaticSegmenter::PunctuationReplacer.new(
+        matches_array: txt.scan(BETWEEN_EM_DASHES_REGEX),
+        text: txt
+      ).replace
+    end
+
     def sub_punctuation_between_quotes_slanted(txt)
       PragmaticSegmenter::PunctuationReplacer.new(
         matches_array: txt.scan(BETWEEN_QUOTE_SLANTED_REGEX),

data/lib/pragmatic_segmenter/version.rb

@@ -1,3 +1,3 @@
 module PragmaticSegmenter
-  VERSION = "0.3.14"
+  VERSION = "0.3.15"
 end

data/spec/pragmatic_segmenter/languages/english_spec.rb

@@ -1384,5 +1384,20 @@ RSpec.describe PragmaticSegmenter::Languages::English, "(en)" do
       ps = PragmaticSegmenter::Segmenter.new(text: "After completion of each Period, I will be paid an advance amount of rs. 1000 and this amount will be deducted from my final study compensation.", clean: false)
       expect(ps.segment).to eq(["After completion of each Period, I will be paid an advance amount of rs. 1000 and this amount will be deducted from my final study compensation."])
     end
+
+    it "correctly segments text #114" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Mix it, put it in the oven, and -- voila! -- you have cake.", clean: false)
+      expect(ps.segment).to eq(["Mix it, put it in the oven, and -- voila! -- you have cake."])
+    end
+
+    it "correctly segments text #115" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Some can be -- if I may say so? -- a bit questionable.", clean: false)
+      expect(ps.segment).to eq(["Some can be -- if I may say so? -- a bit questionable."])
+    end
+
+    it "correctly segments text #116" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "What do you see? - Posted like silent sentinels all around the town, stand thousands upon thousands of mortal men fixed in ocean reveries.", clean: false)
+      expect(ps.segment).to eq(["What do you see?", "- Posted like silent sentinels all around the town, stand thousands upon thousands of mortal men fixed in ocean reveries."])
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.3.14
+  version: 0.3.15
 platform: ruby
 authors:
 - Kevin S. Dias