pragmatic_segmenter 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1cf05fd20f9672f1186e1c4f857f6045d2fcf4b8
-  data.tar.gz: f8c2685f66e2a177f18683bae290dfc892c8b67f
+  metadata.gz: ebda4e8fba93e6cc9fbaeb57fe71e0d3ae47d721
+  data.tar.gz: 3b2e121d592797dea12efc587c198082a4a23955
 SHA512:
-  metadata.gz: b45bc699fa3cb055c54a028f87c803889ac3889ba454534abf794c9fc5cce36983172898955723c0c4b05ec040320e0126fa36fdf565897ca358aa0d34b1db8f
-  data.tar.gz: 65384a1dd4e0b43a03925447e7dbf4ce7c8fe1f4c58ee86c3b660f1a33e3783a1cc9c73235ff600b4c32c140a15a54955eb55a117e83ddf4b0d4064a2616e533
+  metadata.gz: 169f112a82005dcd9b399b8337ef0e25afffde10f6a30f4c038f7621550dd3696c4ad054725763734d420de6445577946c8fc6c249e1d0ff13f5788adde0b9b9
+  data.tar.gz: 9b4b51797b69972e30adc23f948de2a26bc9a9e160ce1271c8f869055e466be83e451bc36eed44020d1e6ee903922d3594ef254e01df337ee884ca80a2a72965

data/README.md

@@ -754,6 +754,9 @@ To test the relative performance of different segmentation tools and libraries I
 * Fix missing abbreviations  
 * Add footnote rule to `cleaner.rb`  
 
+**Version 0.1.3**  
+* Improve punctuation in bracket replacement  
+
 ## Contributing
 
 If you find a text that is incorrectly segmented using this gem, please submit an issue.

data/lib/pragmatic_segmenter/abbreviation_replacer.rb

@@ -136,16 +136,19 @@ module PragmaticSegmenter
     end
 
     def replace_pre_number_abbr(txt, abbr)
-      txt.gsub(/(?<=#{abbr.strip})\.(?=\s\d)/, '∯').gsub(/(?<=#{abbr.strip})\.(?=\s+\()/, '∯')
+      txt.gsub(/(?<=\s#{abbr.strip})\.(?=\s\d)|(?<=^#{abbr.strip})\.(?=\s\d)/, '∯')
+         .gsub(/(?<=\s#{abbr.strip})\.(?=\s+\()|(?<=^#{abbr.strip})\.(?=\s+\()/, '∯')
+
     end
 
     def replace_prepositive_abbr(txt, abbr)
-      txt.gsub(/(?<=#{abbr.strip})\.(?=\s)/, '∯')
+      txt.gsub(/(?<=\s#{abbr.strip})\.(?=\s)|(?<=^#{abbr.strip})\.(?=\s)/, '∯')
+         .gsub(/(?<=\s#{abbr.strip})\.(?=:\d+)|(?<=^#{abbr.strip})\.(?=:\d+)/, '∯')
     end
 
     def replace_period_of_abbr(txt, abbr)
-      txt.gsub(/(?<=#{abbr.strip})\.(?=((\.|:|\?)|(\s([a-z]|I\s|I'm|I'll|\d))))/, '∯')
-         .gsub(/(?<=#{abbr.strip})\.(?=,)/, '∯')
+      txt.gsub(/(?<=\s#{abbr.strip})\.(?=((\.|\:|\?)|(\s([a-z]|I\s|I'm|I'll|\d))))|(?<=^#{abbr.strip})\.(?=((\.|\:|\?)|(\s([a-z]|I\s|I'm|I'll|\d))))/, '∯')
+         .gsub(/(?<=\s#{abbr.strip})\.(?=,)|(?<=^#{abbr.strip})\.(?=,)/, '∯')
     end
 
     def replace_possessive_abbreviations(txt)

data/lib/pragmatic_segmenter/cleaner.rb

@@ -61,9 +61,6 @@ module PragmaticSegmenter
     # Rubular: http://rubular.com/r/IQ4TPfsbd8
     ConsecutiveForwardSlashRule = Rule.new(/\/{3}/, '')
 
-    # Rubular: http://rubular.com/r/gEjxQ0HmSD
-    FootnoteRule = Rule.new(/\[\?\]/, '[&ᓷ&]')
-
     # Rubular: http://rubular.com/r/6dt98uI76u
     NoSpaceBetweenSentencesRule = Rule.new(NO_SPACE_BETWEEN_SENTENCES_REGEX, '. ')
 
@@ -108,7 +105,8 @@ module PragmaticSegmenter
       replace_newlines(@clean_text)
       replace_escaped_newlines(@clean_text)
       @clean_text.apply(HtmlRules::All)
-      @clean_text.apply(InlineFormattingRule, FootnoteRule)
+      replace_punctuation_in_brackets(@clean_text)
+      @clean_text.apply(InlineFormattingRule)
       clean_quotations(@clean_text)
       clean_table_of_contents(@clean_text)
       check_for_no_space_in_between_sentences(@clean_text)
@@ -126,6 +124,12 @@ module PragmaticSegmenter
       txt
     end
 
+    def replace_punctuation_in_brackets(txt)
+      txt.dup.gsub!(/\[(?:[^\]])*\]/) do |match|
+        txt.gsub!(/#{Regexp.escape(match)}/, "#{match.dup.gsub!(/\?/, '&ᓷ&')}") if match.include?('?')
+      end
+    end
+
     def search_for_connected_sentences(word, txt, regex, rule)
       if word =~ regex
         unless URL_EMAIL_KEYWORDS.any? { |web| word =~ /#{web}/ }

data/lib/pragmatic_segmenter/languages/spanish.rb

@@ -19,7 +19,7 @@ module PragmaticSegmenter
 
       class Abbreviation < PragmaticSegmenter::Abbreviation
         ABBREVIATIONS = ['a.c', 'a/c', 'abr', 'adj', 'admón', 'afmo', 'ago', 'almte', 'ap', 'apdo', 'arq', 'art', 'atte', 'av', 'avda', 'bco', 'bibl', 'bs. as', 'c', 'c.f', 'c.g', 'c/c', 'c/u', 'cap', 'cc.aa', 'cdad', 'cm', 'co', 'cra', 'cta', 'cv', 'd.e.p', 'da', 'dcha', 'dcho', 'dep', 'dic', 'dicc', 'dir', 'dn', 'doc', 'dom', 'dpto', 'dr', 'dra', 'dto', 'ee', 'ej', 'en', 'entlo', 'esq', 'etc', 'excmo', 'ext', 'f.c', 'fca', 'fdo', 'febr', 'ff. aa', 'ff.cc', 'fig', 'fil', 'fra', 'g.p', 'g/p', 'gob', 'gr', 'gral', 'grs', 'hnos', 'hs', 'igl', 'iltre', 'imp', 'impr', 'impto', 'incl', 'ing', 'inst', 'izdo', 'izq', 'izqdo', 'j.c', 'jue', 'jul', 'jun', 'kg', 'km', 'lcdo', 'ldo', 'let', 'lic', 'ltd', 'lun', 'mar', 'may', 'mg', 'min', 'mié', 'mm', 'máx', 'mín', 'mt', 'n. del t', 'n.b', 'no', 'nov', 'ntra. sra', 'núm', 'oct', 'p', 'p.a', 'p.d', 'p.ej', 'p.v.p', 'párrf', 'ppal', 'prev', 'prof', 'prov', 'ptas', 'pts', 'pza', 'pág', 'págs', 'párr', 'q.e.g.e', 'q.e.p.d', 'q.e.s.m', 'reg', 'rep', 'rr. hh', 'rte', 's', 's. a', 's.a.r', 's.e', 's.l', 's.r.c', 's.r.l', 's.s.s', 's/n', 'sdad', 'seg', 'sept', 'sig', 'sr', 'sra', 'sres', 'srta', 'sta', 'sto', 'sáb', 't.v.e', 'tamb', 'tel', 'tfno', 'ud', 'uu', 'uds', 'univ', 'v.b', 'v.e', 'vd', 'vds', 'vid', 'vie', 'vol', 'vs', 'vto', 'a', 'aero', 'ambi', 'an', 'anfi', 'ante', 'anti', 'archi', 'arci', 'auto', 'bi', 'bien', 'bis', 'co', 'com', 'con', 'contra', 'crio', 'cuadri', 'cuasi', 'cuatri', 'de', 'deci', 'des', 'di', 'dis', 'dr', 'ecto', 'en', 'endo', 'entre', 'epi', 'equi', 'ex', 'extra', 'geo', 'hemi', 'hetero', 'hiper', 'hipo', 'homo', 'i', 'im', 'in', 'infra', 'inter', 'intra', 'iso', 'lic', 'macro', 'mega', 'micro', 'mini', 'mono', 'multi', 'neo', 'omni', 'para', 'pen', 'ph', 'ph.d', 'pluri', 'poli', 'pos', 'post', 'pre', 'pro', 'pseudo', 're', 'retro', 'semi', 'seudo', 'sobre', 'sub', 'super', 'supra', 'trans', 'tras', 'tri', 'ulter', 'ultra', 'un', 'uni', 'vice', 'yuxta']
-        PREPOSITIVE_ABBREVIATIONS = ['a', 'aero', 'ambi', 'an', 'anfi', 'ante', 'anti', 'archi', 'arci', 'auto', 'bi', 'bien', 'bis', 'co', 'com', 'con', 'contra', 'crio', 'cuadri', 'cuasi', 'cuatri', 'de', 'deci', 'des', 'di', 'dis', 'dr', 'ecto', 'ee', 'en', 'endo', 'entre', 'epi', 'equi', 'ex', 'extra', 'geo', 'hemi', 'hetero', 'hiper', 'hipo', 'homo', 'i', 'im', 'in', 'infra', 'inter', 'intra', 'iso', 'lic', 'macro', 'mega', 'micro', 'mini', 'mono', 'mt', 'multi', 'neo', 'omni', 'para', 'pen', 'ph', 'pluri', 'poli', 'pos', 'post', 'pre', 'pro', 'prof', 'pseudo', 're', 'retro', 'semi', 'seudo', 'sobre', 'sub', 'super', 'supra', 'srta', 'trans', 'tras', 'tri', 'ulter', 'ultra', 'un', 'uni', 'vice', 'yuxta']
+        PREPOSITIVE_ABBREVIATIONS = ['a', 'aero', 'ambi', 'an', 'anfi', 'ante', 'anti', 'archi', 'arci', 'auto', 'bi', 'bien', 'bis', 'co', 'com', 'con', 'contra', 'crio', 'cuadri', 'cuasi', 'cuatri', 'de', 'deci', 'des', 'di', 'dis', 'dr', 'ecto', 'ee', 'en', 'endo', 'entre', 'epi', 'equi', 'ex', 'extra', 'geo', 'hemi', 'hetero', 'hiper', 'hipo', 'homo', 'i', 'im', 'in', 'infra', 'inter', 'intra', 'iso', 'lic', 'macro', 'mega', 'micro', 'mini', 'mono', 'mt', 'multi', 'neo', 'omni', 'para', 'pen', 'ph', 'pluri', 'poli', 'pos', 'post', 'pre', 'pro', 'prof', 'pseudo', 're', 'retro', 'semi', 'seudo', 'sobre', 'sub', 'super', 'supra', 'sra', 'srta', 'trans', 'tras', 'tri', 'ulter', 'ultra', 'un', 'uni', 'vice', 'yuxta']
         NUMBER_ABBREVIATIONS = ['cra', 'ext', 'no', 'nos', 'p', 'pp', 'tel']
 
         def all

data/lib/pragmatic_segmenter/version.rb

@@ -1,3 +1,3 @@
 module PragmaticSegmenter
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/spec/pragmatic_segmenter_spec.rb

@@ -923,6 +923,11 @@ RSpec.describe PragmaticSegmenter::Segmenter do
         ps = PragmaticSegmenter::Segmenter.new(text: "[?][footnoteRef:6] This is a footnote.")
         expect(ps.segment).to eq(["[?][footnoteRef:6] This is a footnote."])
       end
+
+      it "correctly segments text #091" do
+        ps = PragmaticSegmenter::Segmenter.new(text: "[15:  12:32]  [16:  firma? 13:28]")
+        expect(ps.segment).to eq(["[15:  12:32]  [16:  firma? 13:28]"])
+      end
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Kevin S. Dias