pragmatic_segmenter 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a2951fa2242e1eb7ce0898862d41fd5239d871f8
-  data.tar.gz: 01d84b2637e84598907ef08e59ec64e90855c2ec
+  metadata.gz: 3660e1361fcdab9e88c4a53530ee3c6b63afce92
+  data.tar.gz: baf77669ff3b7ee9d5c5172206ed79c6ec966948
 SHA512:
-  metadata.gz: 25ec02bc41c649a57b1bc993848bed546514568c74e2e0ec5b6a2242fbbb579e61e199210f8ea4349ceadee5a06ad0fa76c7c60196762903cab984bbad3c2286
-  data.tar.gz: 925a24d8c131813ddb4f80a8865697baa0efe05ef781cabfc13dfb9b1efc878c2e6e276a46a9f1301049a99b2f64a558b6a9911e0a2f5f8f4c17622df3fd0b2a
+  metadata.gz: 901a24ad1a48ebca4e2daf88bc593e8b82e9fd1f8c7eee28ff81de4c32757571dc549e439d9fd6bba557846f3428e57a8290acba6d524516b237830cea9cdc7c
+  data.tar.gz: 9a83f91f6e3a6d22d97498173c486b7e0a608563f3ed77091c409510242e7252db1fed6a448bd29089fdb5da41c0dd0f21660c6a97c35e87862928775d9f7abe

data/README.md

@@ -761,7 +761,10 @@ To test the relative performance of different segmentation tools and libraries I
 * Fix missing abbreviations  
 
 **Version 0.1.5**  
-* Fix comma at end of quoatation bug  
+* Fix comma at end of quotation bug  
+
+**Version 0.1.6**  
+* Fix bug in numbered list finder (ignore longer digits)   
 
 ## Contributing
 

data/lib/pragmatic_segmenter/list.rb

@@ -16,19 +16,19 @@ module PragmaticSegmenter
     ListMarkerRule = Rule.new(/☝/, '')
 
     # Rubular: http://rubular.com/r/Wv4qLdoPx7
-    SpaceBetweenListItemsFirstRule = Rule.new(/(?<=\S\S|^)\s(?=\S\s*\d+♨)/, "\r")
+    SpaceBetweenListItemsFirstRule = Rule.new(/(?<=\S\S|^)\s(?=\S\s*\d{1,2}♨)/, "\r")
 
     # Rubular: http://rubular.com/r/AizHXC6HxK
-    SpaceBetweenListItemsSecondRule = Rule.new(/(?<=\S\S|^)\s(?=\d+♨)/, "\r")
+    SpaceBetweenListItemsSecondRule = Rule.new(/(?<=\S\S|^)\s(?=\d{1,2}♨)/, "\r")
 
     # Rubular: http://rubular.com/r/GE5q6yID2j
-    SpaceBetweenListItemsThirdRule = Rule.new(/(?<=\S\S|^)\s(?=\d+☝)/, "\r")
+    SpaceBetweenListItemsThirdRule = Rule.new(/(?<=\S\S|^)\s(?=\d{1,2}☝)/, "\r")
 
     NUMBERED_LIST_REGEX_1 =
-      /\s\d+(?=\.\s)|^\d+(?=\.\s)|\s\d+(?=\.\))|^\d+(?=\.\))|(?<=\s\-)\d+(?=\.\s)|(?<=^\-)\d+(?=\.\s)|(?<=\s\⁃)\d+(?=\.\s)|(?<=^\⁃)\d+(?=\.\s)|(?<=s\-)\d+(?=\.\))|(?<=^\-)\d+(?=\.\))|(?<=\s\⁃)\d+(?=\.\))|(?<=^\⁃)\d+(?=\.\))/
+      /\s\d{1,2}(?=\.\s)|^\d{1,2}(?=\.\s)|\s\d{1,2}(?=\.\))|^\d{1,2}(?=\.\))|(?<=\s\-)\d{1,2}(?=\.\s)|(?<=^\-)\d{1,2}(?=\.\s)|(?<=\s\⁃)\d{1,2}(?=\.\s)|(?<=^\⁃)\d{1,2}(?=\.\s)|(?<=s\-)\d{1,2}(?=\.\))|(?<=^\-)\d{1,2}(?=\.\))|(?<=\s\⁃)\d{1,2}(?=\.\))|(?<=^\⁃)\d{1,2}(?=\.\))/
     NUMBERED_LIST_REGEX_2 =
-      /(?<=\s)\d+\.(?=\s)|^\d+\.(?=\s)|(?<=\s)\d+\.(?=\))|^\d+\.(?=\))|(?<=\s\-)\d+\.(?=\s)|(?<=^\-)\d+\.(?=\s)|(?<=\s\⁃)\d+\.(?=\s)|(?<=^\⁃)\d+\.(?=\s)|(?<=\s\-)\d+\.(?=\))|(?<=^\-)\d+\.(?=\))|(?<=\s\⁃)\d+\.(?=\))|(?<=^\⁃)\d+\.(?=\))/
-    NUMBERED_LIST_PARENS_REGEX = /\d+(?=\)\s)/
+      /(?<=\s)\d{1,2}\.(?=\s)|^\d{1,2}\.(?=\s)|(?<=\s)\d{1,2}\.(?=\))|^\d{1,2}\.(?=\))|(?<=\s\-)\d{1,2}\.(?=\s)|(?<=^\-)\d{1,2}\.(?=\s)|(?<=\s\⁃)\d{1,2}\.(?=\s)|(?<=^\⁃)\d{1,2}\.(?=\s)|(?<=\s\-)\d{1,2}\.(?=\))|(?<=^\-)\d{1,2}\.(?=\))|(?<=\s\⁃)\d{1,2}\.(?=\))|(?<=^\⁃)\d{1,2}\.(?=\))/
+    NUMBERED_LIST_PARENS_REGEX = /\d{1,2}(?=\)\s)/
 
     # Rubular: http://rubular.com/r/NsNFSqrNvJ
     EXTRACT_ALPHABETICAL_LIST_LETTERS_REGEX =
@@ -81,7 +81,7 @@ module PragmaticSegmenter
     def add_line_breaks_for_numbered_list_with_periods(txt)
       return txt unless txt.include?('♨') &&
                         txt !~ /♨.+\n.+♨|♨.+\r.+♨/ &&
-                        txt !~ /for\s\d+♨\s[a-z]/
+                        txt !~ /for\s\d{1,2}♨\s[a-z]/
       txt.apply(SpaceBetweenListItemsFirstRule).
           apply(SpaceBetweenListItemsSecondRule)
     end

data/lib/pragmatic_segmenter/version.rb

@@ -1,3 +1,3 @@
 module PragmaticSegmenter
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

data/spec/pragmatic_segmenter_spec.rb

@@ -933,6 +933,11 @@ RSpec.describe PragmaticSegmenter::Segmenter do
         ps = PragmaticSegmenter::Segmenter.new(text: "\"It's a good thing that the water is really calm,\" I answered ironically.")
         expect(ps.segment).to eq(["\"It's a good thing that the water is really calm,\" I answered ironically."])
       end
+
+      it "correctly segments text #093" do
+        ps = PragmaticSegmenter::Segmenter.new(text: "December 31, 1988. Hello world. It's great! \nBorn April 05, 1989.")
+        expect(ps.segment).to eq(["December 31, 1988.", "Hello world.", "It's great!", "Born April 05, 1989."])
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-13 00:00:00.000000000 Z
+date: 2015-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler