pragmatic_segmenter 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 80eb7dfc7aeed8a66ff324dde7a87ea544e55e45
-  data.tar.gz: c1f3daf78133d748d6cf02133937dad1e05ef0ee
+  metadata.gz: c54f1c34c1f0bd34858cbd9c3915ec47f8a4d807
+  data.tar.gz: 19a936bfd8fb6c0046f06a20f50319fb5a2c0ed7
 SHA512:
-  metadata.gz: f2c6018bb4d46ccc5ef86bbc548437861fa83cf4948abd09e45d8c4b23c021c48907feb9cd65916a9ef8635ede477f5f0b8e221cb7452b38c236f79a3d0cfa77
-  data.tar.gz: d63e6eb39e52306785e491f1bfd1d4196e992e513a857b8893ef9c24fcab95f515dc863789f5040609436dc116efbac608a7ac010c0015fa159535198b1554ea
+  metadata.gz: 824392373d56549289ae89de98976d53afdf74ccde45f3b6069164d8882fbeef7ae04e2fec1261af2baf43223246b4f1f90ebab12b97fa0c22e0d81d6caa56fa
+  data.tar.gz: 4f43e20c69be0515d81c9d98e33b7ed2aadf05d5110ddd7acae3f971745c8406c03763fdc275afa3b14de3d5aa8846a85638845d90cc4c9fd34f4cf9e120d1d3

data/README.md

@@ -741,6 +741,9 @@ To test the relative performance of different segmentation tools and libraries I
 **Version 0.0.8**  
 * Fix error in `list.rb`  
 
+**Version 0.0.9**  
+* Improve handling of alphabetical and roman numeral lists  
+
 ## Contributing
 
 If you find a text that is incorrectly segmented using this gem, please submit an issue.

data/lib/pragmatic_segmenter/list.rb

@@ -32,7 +32,7 @@ module PragmaticSegmenter
 
     # Rubular: http://rubular.com/r/NsNFSqrNvJ
     EXTRACT_ALPHABETICAL_LIST_LETTERS_REGEX =
-      /(?<=\()[a-z]+(?=\))|(?<=^)[a-z]+(?=\))|(?<=\A)[a-z]+(?=\))|(?<=\s)[a-z]+(?=\))/i
+      /\([a-z]+(?=\))|(?<=^)[a-z]+(?=\))|(?<=\A)[a-z]+(?=\))|(?<=\s)[a-z]+(?=\))/i
 
     # Rubular: http://rubular.com/r/wMpnVedEIb
     ALPHABETICAL_LIST_LETTERS_AND_PERIODS_REGEX =
@@ -134,10 +134,10 @@ module PragmaticSegmenter
 
     def replace_alphabet_list_parens(a, txt)
       txt.gsub!(EXTRACT_ALPHABETICAL_LIST_LETTERS_REGEX).with_index do |m|
-        if txt =~ /\(#{Regexp.escape(m.to_s)}\)/i
-          a.eql?(m.dup.downcase) ? "\rȸ(#{Regexp.escape(m.to_s)}" : "#{m}"
+        if m.include?('(')
+          a.eql?(m.dup.downcase.gsub!(/\(/, '')) ? "\r&✂&#{Regexp.escape(m.gsub!(/\(/, ''))}" : "#{m}"
         else
-          a.eql?(m.dup.downcase) ? "\r#{Regexp.escape(m.to_s)}" : "#{m}"
+          a.eql?(m.dup.downcase) ? "\r#{Regexp.escape(m)}" : "#{m}"
         end
       end
     end
@@ -175,6 +175,7 @@ module PragmaticSegmenter
       else
         alphabet = ('a'..'z').to_a
       end
+      list_array.delete_if { |item| !alphabet.any? { |a| a.include?(item) } }
       list_array.each_with_index do |a, i|
         if i.eql?(list_array.length - 1)
           last_array_item_replacement(a, i, alphabet, list_array, txt, parens)

data/lib/pragmatic_segmenter/rules.rb

@@ -61,6 +61,7 @@ module PragmaticSegmenter
       MixedDoubleQQ = Rule.new(/☇/, '??')
       MixedDoubleEQ = Rule.new(/☈/, '!?')
       MixedDoubleEE = Rule.new(/☄/, '!!')
+      LeftParens = Rule.new(/&✂&/, '(')
       TemporaryEndingPunctutation = Rule.new('ȸ', '')
       Newline = Rule.new(/ȹ/, "\n")
 
@@ -70,7 +71,8 @@ module PragmaticSegmenter
               ExclamationPoint, QuestionMark,
               FullWidthQuestionMark, MixedDoubleQE,
               MixedDoubleQQ, MixedDoubleEQ,
-              MixedDoubleEE, TemporaryEndingPunctutation,
+              MixedDoubleEE, LeftParens,
+              TemporaryEndingPunctutation,
               Newline ]
     end
 

data/lib/pragmatic_segmenter/sentence_boundary_punctuation.rb

@@ -3,7 +3,7 @@
 module PragmaticSegmenter
   # This class splits text at sentence boundary punctuation marks
   class SentenceBoundaryPunctuation
-    SENTENCE_BOUNDARY_REGEX = /\u{ff08}(?:[^\u{ff09}])*\u{ff09}(?=\s?[A-Z])|\u{300c}(?:[^\u{300d}])*\u{300d}(?=\s[A-Z])|\((?:[^\)])*\)(?=\s[A-Z])|'(?:[^'])*'(?=\s[A-Z])|"(?:[^"])*"(?=\s[A-Z])|“(?:[^”])*”(?=\s[A-Z])|\S.*?[。．.！!?？ȸȹ☉☈☇☄]/
+    SENTENCE_BOUNDARY_REGEX = /\u{ff08}(?:[^\u{ff09}])*\u{ff09}(?=\s?[A-Z])|\u{300c}(?:[^\u{300d}])*\u{300d}(?=\s[A-Z])|\((?:[^\)]){2,}\)(?=\s[A-Z])|'(?:[^'])*'(?=\s[A-Z])|"(?:[^"])*"(?=\s[A-Z])|“(?:[^”])*”(?=\s[A-Z])|\S.*?[。．.！!?？ȸȹ☉☈☇☄]/
 
     attr_reader :text
     def initialize(text:)

data/lib/pragmatic_segmenter/version.rb

@@ -1,3 +1,3 @@
 module PragmaticSegmenter
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/spec/pragmatic_segmenter_spec.rb

@@ -900,7 +900,7 @@ RSpec.describe PragmaticSegmenter::Segmenter do
       end
 
       it "correctly segments text #086" do
-        ps = PragmaticSegmenter::Segmenter.new(text: "(a) Hello world. \n(b) Hello world.\n(c) Hello world.\n(d) Hello world.\n(e) Hello world.\n(f) Hello world.", language: "en")
+        ps = PragmaticSegmenter::Segmenter.new(text: "(a) Hello world. (b) Hello world. (c) Hello world. (d) Hello world. (e) Hello world.\n(f) Hello world.", language: "en")
         expect(ps.segment).to eq(["(a) Hello world.", "(b) Hello world.", "(c) Hello world.", "(d) Hello world.", "(e) Hello world.", "(f) Hello world."])
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Kevin S. Dias