pragmatic_segmenter 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 805bb57215b38dc30f107be60c18c141d8dc1297
-  data.tar.gz: 389bf5f6700f44cb255279a4391c849d0ac67f69
+  metadata.gz: f8c68d5563d388488aeacf96083dc2c81191b364
+  data.tar.gz: 60f67ff5dc22c136f389f48ff9ba76350de013df
 SHA512:
-  metadata.gz: 749c13c831913bcd606f0d08194d09ca982c7e033c59c8520bf1b20a4146934725ab8252bc54362815cd34f5ea45f8c5ed1cde2da80b8beb506093bd86138522
-  data.tar.gz: d5f05492994604c023fc357b7b030878eb2d8554f2d898efe196530a5777d394359acd4e8c600451cec511df4e2dfafa30cc34d2589a8d983ae3b11299aa9869
+  metadata.gz: 3dcc1aa9da843232653928fb1a961f1b9d053aa9556924c4bed109a4c250c32bf1f11ccd69bdef6e6e1f40e3293e14d6274b06bd689cd03fb50c155200f29a98
+  data.tar.gz: 5b0220d3d9645a78025bdd76b9bff39611255de9e8d568beb053d738e42152ada640bd3d973e6f833ef77e5ff66f12ecd7dd1450617062f7d37fb558ff25ad28

data/README.md

@@ -817,7 +817,16 @@ To test the relative performance of different segmentation tools and libraries I
 * Add English abbreviations  
 
 **Version 0.3.3**  
-* Fix cleaner bug    
+* Fix cleaner bug  
+
+**Version 0.3.4**  
+* Large refactor   
+
+**Version 0.3.5**  
+* Reduce GC by replacing #gusb with #gsub! where possible  
+
+**Version 0.3.6**  
+* Refactor SENTENCE_STARTERS to each individual language and add SENTENCE_STARTERS for German    
 
 ## Contributing
 

data/lib/pragmatic_segmenter/abbreviation_replacer.rb

@@ -5,8 +5,6 @@ module PragmaticSegmenter
   # replaces the periods.
   class AbbreviationReplacer
 
-    SENTENCE_STARTERS = %w(A Being Did For He How However I In It Millions More She That The There They We What When Where Who Why)
-
     attr_reader :text
     def initialize(text:, language: )
       @text = Text.new(text)
@@ -75,18 +73,19 @@ module PragmaticSegmenter
       # and try to cover the words that most often start a
       # sentence but could never follow one of the abbreviations below.
 
-      SENTENCE_STARTERS.each do |word|
-        txt.gsub!(/U∯S∯\s#{Regexp.escape(word)}\s/, "U∯S\.\s#{Regexp.escape(word)}\s")
-        txt.gsub!(/U\.S∯\s#{Regexp.escape(word)}\s/, "U\.S\.\s#{Regexp.escape(word)}\s")
-        txt.gsub!(/U∯K∯\s#{Regexp.escape(word)}\s/, "U∯K\.\s#{Regexp.escape(word)}\s")
-        txt.gsub!(/U\.K∯\s#{Regexp.escape(word)}\s/, "U\.K\.\s#{Regexp.escape(word)}\s")
-        txt.gsub!(/E∯U∯\s#{Regexp.escape(word)}\s/, "E∯U\.\s#{Regexp.escape(word)}\s")
-        txt.gsub!(/E\.U∯\s#{Regexp.escape(word)}\s/, "E\.U\.\s#{Regexp.escape(word)}\s")
-        txt.gsub!(/U∯S∯A∯\s#{Regexp.escape(word)}\s/, "U∯S∯A\.\s#{Regexp.escape(word)}\s")
-        txt.gsub!(/U\.S\.A∯\s#{Regexp.escape(word)}\s/, "U\.S\.A\.\s#{Regexp.escape(word)}\s")
-        txt.gsub!(/I∯\s#{Regexp.escape(word)}\s/, "I\.\s#{Regexp.escape(word)}\s")
-        txt.gsub!(/i.v∯\s#{Regexp.escape(word)}\s/, "i\.v\.\s#{Regexp.escape(word)}\s")
-        txt.gsub!(/I.V∯\s#{Regexp.escape(word)}\s/, "I\.V\.\s#{Regexp.escape(word)}\s")
+      @language::AbbreviationReplacer::SENTENCE_STARTERS.each do |word|
+        escaped = Regexp.escape(word)
+        txt.gsub!(/U∯S∯\s#{escaped}\s/, "U∯S\.\s#{escaped}\s")
+        txt.gsub!(/U\.S∯\s#{escaped}\s/, "U\.S\.\s#{escaped}\s")
+        txt.gsub!(/U∯K∯\s#{escaped}\s/, "U∯K\.\s#{escaped}\s")
+        txt.gsub!(/U\.K∯\s#{escaped}\s/, "U\.K\.\s#{escaped}\s")
+        txt.gsub!(/E∯U∯\s#{escaped}\s/, "E∯U\.\s#{escaped}\s")
+        txt.gsub!(/E\.U∯\s#{escaped}\s/, "E\.U\.\s#{escaped}\s")
+        txt.gsub!(/U∯S∯A∯\s#{escaped}\s/, "U∯S∯A\.\s#{escaped}\s")
+        txt.gsub!(/U\.S\.A∯\s#{escaped}\s/, "U\.S\.A\.\s#{escaped}\s")
+        txt.gsub!(/I∯\s#{escaped}\s/, "I\.\s#{escaped}\s")
+        txt.gsub!(/i.v∯\s#{escaped}\s/, "i\.v\.\s#{escaped}\s")
+        txt.gsub!(/I.V∯\s#{escaped}\s/, "I\.V\.\s#{escaped}\s")
       end
       txt
     end

data/lib/pragmatic_segmenter/languages/amharic.rb

@@ -5,6 +5,10 @@ module PragmaticSegmenter
 
       SENTENCE_BOUNDARY_REGEX = /.*?[፧።!\?]|.*?$/
       Punctuations = ['።', '፧', '?', '!']
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/arabic.rb

@@ -18,7 +18,8 @@ module PragmaticSegmenter
       # Rubular: http://rubular.com/r/kPRgApNHUg
       ReplaceNonSentenceBoundaryCommaRule = Rule.new(/،(?=\s\S+،)/, '♬')
 
-      class AbbreviationReplacer  < AbbreviationReplacer
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
         private
 
         def scan_for_replacements(txt, am, index, character_array)

data/lib/pragmatic_segmenter/languages/armenian.rb

@@ -5,6 +5,10 @@ module PragmaticSegmenter
 
       SENTENCE_BOUNDARY_REGEX = /.*?[։՜:]|.*?$/
       Punctuations = ['։', '՜', ':']
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/burmese.rb

@@ -5,6 +5,10 @@ module PragmaticSegmenter
 
       SENTENCE_BOUNDARY_REGEX = /.*?[။၏!\?]|.*?$/
       Punctuations = ['။', '၏', '?', '!']
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/chinese.rb

@@ -3,6 +3,9 @@ module PragmaticSegmenter
     module Chinese
       include Languages::Common
 
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/common.rb

@@ -97,6 +97,14 @@ module PragmaticSegmenter
       ExtraWhiteSpaceRule = Rule.new(/\s{3,}/, ' ')
 
       SubSingleQuoteRule = Rule.new(/&⎋&/, "'")
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = %w(
+          A Being Did For He How However I In It Millions More She That The
+          There They We What When Where Who Why
+        ).freeze
+      end
+
     end
   end
 end

data/lib/pragmatic_segmenter/languages/deutsch.rb

@@ -58,7 +58,13 @@ module PragmaticSegmenter
         end
       end
 
-      class AbbreviationReplacer  < AbbreviationReplacer
+      class AbbreviationReplacer < AbbreviationReplacer
+
+        SENTENCE_STARTERS = %w(
+          Am Auch Auf Bei Da Das Der Die Ein Eine Es Für Heute Ich Im In
+          Ist Jetzt Mein Mit Nach So Und Warum Was Wenn Wer Wie Wir
+        ).freeze
+
         def replace
           @text = text.apply(
             @language::PossessiveAbbreviationRule,

data/lib/pragmatic_segmenter/languages/english.rb

@@ -19,6 +19,13 @@ module PragmaticSegmenter
           []
         end
       end
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = %w(
+          A Being Did For He How However I In It Millions More She That The
+          There They We What When Where Who Why
+        ).freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/french.rb

@@ -8,6 +8,10 @@ module PragmaticSegmenter
         PREPOSITIVE_ABBREVIATIONS = []
         NUMBER_ABBREVIATIONS = []
       end
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/greek.rb

@@ -5,6 +5,10 @@ module PragmaticSegmenter
 
       SENTENCE_BOUNDARY_REGEX = /.*?[\.;!\?]|.*?$/
       Punctuations = ['.', '!', ';', '?']
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/hindi.rb

@@ -5,6 +5,10 @@ module PragmaticSegmenter
 
       SENTENCE_BOUNDARY_REGEX = /.*?[।\|!\?]|.*?$/
       Punctuations = ['।', '|', '.', '!', '?']
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/italian.rb

@@ -8,6 +8,10 @@ module PragmaticSegmenter
         PREPOSITIVE_ABBREVIATIONS = ['a.c', 'acc', 'adj', 'adm', 'adv', 'all', 'amn', 'arch', 'asst', 'avv', 'banc', 'bart', 'bcc', 'bldg', 'brig', 'bros', 'c.a', 'c.a.p', 'c.c.p', 'c.m', 'c.p', 'c.p', 'c.s', 'c.v', 'capt', 'cc', 'cmdr', 'co', 'col', 'comdr', 'con', 'corp', 'corr', 'cpl', 'dir', 'dott', 'dott', 'dr', 'dr', 'drs', 'e.p.c', 'ecc', 'egr', 'ens', 'es', 'fatt', 'gen', 'geom', 'gg', 'gov', 'hon', 'hosp', 'hr', 'id', 'ing', 'insp', 'int', "l'avv", "l'ing", 'lett', 'lt', 'maj', 'messrs', 'mlle', 'mm', 'mme', 'mo', 'mons', 'mr', 'mr', 'mrs', 'mrs', 'ms', 'ms', 'msgr', 'n.b', 'ogg', 'on', 'op', 'ord', 'p.c', 'p.c.c', 'p.es', 'p.f', 'p.r', 'p.s', 'p.t', 'p.v', 'pfc', 'ph', 'post', 'pp', 'prof', 'psicol', 'pvt', 'racc', 'rag', 'rep', 'reps', 'res', 'rev', 'ric', 'rif', 'rp', 'rsvp', 'rt', 's.a', 's.b.f', 's.n.c', 's.p.a', 's.p.m', 's.r.l', 'seg', 'sen', 'sens', 'sfc', 'sgg', 'sgt', 'sig', 'sigg', 'soc', 'spett', 'sr', 'ss', 'st', 'supt', 'surg', 'tel', 'u.s', 'v.p', 'v.r', 'v.s']
         NUMBER_ABBREVIATIONS = ['art', 'no', 'nos', 'nr', 'pp']
       end
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/japanese.rb

@@ -19,6 +19,10 @@ module PragmaticSegmenter
         end
       end
 
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
+
       class BetweenPunctuation < PragmaticSegmenter::BetweenPunctuation
         # Rubular: http://rubular.com/r/GnjOmry5Z2
         BETWEEN_QUOTE_JA_REGEX = /\u{300c}(?>[^\u{300c}\u{300d}\\]+|\\{2}|\\.)*\u{300d}/

data/lib/pragmatic_segmenter/languages/persian.rb

@@ -9,7 +9,9 @@ module PragmaticSegmenter
       ReplaceColonBetweenNumbersRule = Rule.new(/(?<=\d):(?=\d)/, '♭')
       ReplaceNonSentenceBoundaryCommaRule = Rule.new(/،(?=\s\S+،)/, '♬')
 
-      class AbbreviationReplacer  < AbbreviationReplacer
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+
         private
 
         def scan_for_replacements(txt, am, index, character_array)

data/lib/pragmatic_segmenter/languages/polish.rb

@@ -8,6 +8,10 @@ module PragmaticSegmenter
         PREPOSITIVE_ABBREVIATIONS = []
         NUMBER_ABBREVIATIONS = []
       end
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/russian.rb

@@ -9,7 +9,9 @@ module PragmaticSegmenter
         NUMBER_ABBREVIATIONS = []
       end
 
-      class AbbreviationReplacer  < AbbreviationReplacer
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+
         private
 
         def replace_period_of_abbr(txt, abbr)

data/lib/pragmatic_segmenter/languages/spanish.rb

@@ -8,6 +8,10 @@ module PragmaticSegmenter
         PREPOSITIVE_ABBREVIATIONS = ['a', 'aero', 'ambi', 'an', 'anfi', 'ante', 'anti', 'archi', 'arci', 'auto', 'bi', 'bien', 'bis', 'co', 'com', 'con', 'contra', 'crio', 'cuadri', 'cuasi', 'cuatri', 'de', 'deci', 'des', 'di', 'dis', 'dr', 'ecto', 'ee', 'en', 'endo', 'entre', 'epi', 'equi', 'ex', 'extra', 'geo', 'hemi', 'hetero', 'hiper', 'hipo', 'homo', 'i', 'im', 'in', 'infra', 'inter', 'intra', 'iso', 'lic', 'macro', 'mega', 'micro', 'mini', 'mono', 'mt', 'multi', 'neo', 'omni', 'para', 'pen', 'ph', 'ph.d', 'pluri', 'poli', 'pos', 'post', 'pre', 'pro', 'prof', 'pseudo', 're', 'retro', 'semi', 'seudo', 'sobre', 'sub', 'super', 'supra', 'sra', 'srta', 'trans', 'tras', 'tri', 'ulter', 'ultra', 'un', 'uni', 'vice', 'yuxta']
         NUMBER_ABBREVIATIONS = ['cra', 'ext', 'no', 'nos', 'p', 'pp', 'tel']
       end
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/urdu.rb

@@ -5,6 +5,10 @@ module PragmaticSegmenter
 
       SENTENCE_BOUNDARY_REGEX = /.*?[۔؟!\?]|.*?$/
       Punctuations = ['?', '!', '۔', '؟']
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/version.rb

@@ -1,3 +1,3 @@
 module PragmaticSegmenter
-  VERSION = "0.3.5"
+  VERSION = "0.3.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-04 00:00:00.000000000 Z
+date: 2016-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler