pragmatic_segmenter 0.0.1

data/lib/pragmatic_segmenter/languages/amharic.rb

@@ -0,0 +1,33 @@
+module PragmaticSegmenter
+  module Languages
+    class Amharic
+      class Process < PragmaticSegmenter::Process
+        private
+
+        def sentence_boundary_punctuation(txt)
+          PragmaticSegmenter::Languages::Amharic::SentenceBoundaryPunctuation.new(text: txt).split
+        end
+
+        def punctuation_array
+          PragmaticSegmenter::Languages::Amharic::Punctuation.new.punct
+        end
+      end
+
+      class SentenceBoundaryPunctuation < PragmaticSegmenter::SentenceBoundaryPunctuation
+        SENTENCE_BOUNDARY = /.*?[፧።!\?]|.*?$/
+
+        def split
+          text.scan(SENTENCE_BOUNDARY)
+        end
+      end
+
+      class Punctuation < PragmaticSegmenter::Punctuation
+        PUNCT = ['።', '፧', '?', '!']
+
+        def punct
+          PUNCT
+        end
+      end
+    end
+  end
+end

data/lib/pragmatic_segmenter/languages/arabic.rb

@@ -0,0 +1,83 @@
+module PragmaticSegmenter
+  module Languages
+    class Arabic
+      class Process < PragmaticSegmenter::Process
+        private
+
+        def sentence_boundary_punctuation(txt)
+          PragmaticSegmenter::Languages::Arabic::SentenceBoundaryPunctuation.new(text: txt).split
+        end
+
+        def replace_abbreviations(txt)
+          PragmaticSegmenter::Languages::Arabic::AbbreviationReplacer.new(text: txt).replace
+        end
+
+        def punctuation_array
+          PragmaticSegmenter::Languages::Arabic::Punctuation.new.punct
+        end
+      end
+
+      class SentenceBoundaryPunctuation < PragmaticSegmenter::SentenceBoundaryPunctuation
+        SENTENCE_BOUNDARY = /.*?[:\.!\?؟،]|.*?\z|.*?$/
+
+        # Rubular: http://rubular.com/r/RX5HpdDIyv
+        ReplaceColonBetweenNumbersRule = Rule.new(/(?<=\d):(?=\d)/, '♭')
+
+        # Rubular: http://rubular.com/r/kPRgApNHUg
+        ReplaceNonSentenceBoundaryCommaRule = Rule.new(/،(?=\s\S+،)/, '♬')
+
+        def split
+          txt = replace_non_sentence_boundary_punctuation(text)
+          txt.scan(SENTENCE_BOUNDARY)
+        end
+
+        private
+
+        def replace_non_sentence_boundary_punctuation(txt)
+          txt.apply(ReplaceColonBetweenNumbersRule).
+              apply(ReplaceNonSentenceBoundaryCommaRule)
+        end
+      end
+
+      class Abbreviation < PragmaticSegmenter::Abbreviation
+        ABBREVIATIONS = ['ا', 'ا. د', 'ا.د', 'ا.ش.ا', 'ا.ش.ا', 'إلخ', 'ت.ب', 'ت.ب', 'ج.ب', 'جم', 'ج.ب', 'ج.م.ع', 'ج.م.ع', 'س.ت', 'س.ت', 'سم', 'ص.ب.', 'ص.ب', 'كج.', 'كلم.', 'م', 'م.ب', 'م.ب', 'ه', 'د‪']
+
+        def all
+          ABBREVIATIONS
+        end
+
+        def prepositive
+          []
+        end
+
+        def number
+          []
+        end
+      end
+
+      class Punctuation < PragmaticSegmenter::Punctuation
+        PUNCT = ['?', '!', ':', '.', '؟', '،']
+
+        def punct
+          PUNCT
+        end
+      end
+
+      class AbbreviationReplacer  < PragmaticSegmenter::AbbreviationReplacer
+        private
+
+        def scan_for_replacements(txt, am, index, character_array, abbr)
+          replace_abbr(txt, am)
+        end
+
+        def replace_abbr(txt, abbr)
+          txt.gsub(/(?<=#{abbr})\./, '∯')
+        end
+
+        def abbreviations
+          PragmaticSegmenter::Languages::Arabic::Abbreviation.new
+        end
+      end
+    end
+  end
+end

data/lib/pragmatic_segmenter/languages/armenian.rb

@@ -0,0 +1,33 @@
+module PragmaticSegmenter
+  module Languages
+    class Armenian
+      class Process < PragmaticSegmenter::Process
+        private
+
+        def sentence_boundary_punctuation(txt)
+          PragmaticSegmenter::Languages::Armenian::SentenceBoundaryPunctuation.new(text: txt).split
+        end
+
+        def punctuation_array
+          PragmaticSegmenter::Languages::Armenian::Punctuation.new.punct
+        end
+      end
+
+      class SentenceBoundaryPunctuation < PragmaticSegmenter::SentenceBoundaryPunctuation
+        SENTENCE_BOUNDARY = /.*?[։՜:]|.*?$/
+
+        def split
+          text.scan(SENTENCE_BOUNDARY)
+        end
+      end
+
+      class Punctuation < PragmaticSegmenter::Punctuation
+        PUNCT = ['։', '՜', ':']
+
+        def punct
+          PUNCT
+        end
+      end
+    end
+  end
+end

data/lib/pragmatic_segmenter/languages/burmese.rb

@@ -0,0 +1,33 @@
+module PragmaticSegmenter
+  module Languages
+    class Burmese
+      class Process < PragmaticSegmenter::Process
+        private
+
+        def sentence_boundary_punctuation(txt)
+          PragmaticSegmenter::Languages::Burmese::SentenceBoundaryPunctuation.new(text: txt).split
+        end
+
+        def punctuation_array
+          PragmaticSegmenter::Languages::Burmese::Punctuation.new.punct
+        end
+      end
+
+      class SentenceBoundaryPunctuation < PragmaticSegmenter::SentenceBoundaryPunctuation
+        SENTENCE_BOUNDARY = /.*?[။၏!\?]|.*?$/
+
+        def split
+          text.scan(SENTENCE_BOUNDARY)
+        end
+      end
+
+      class Punctuation < PragmaticSegmenter::Punctuation
+        PUNCT = ['။', '၏', '?', '!']
+
+        def punct
+          PUNCT
+        end
+      end
+    end
+  end
+end

data/lib/pragmatic_segmenter/languages/deutsch.rb

@@ -0,0 +1,132 @@
+module PragmaticSegmenter
+  module Languages
+    class Deutsch
+      class Process < PragmaticSegmenter::Process
+        private
+
+        def between_punctutation(txt)
+          PragmaticSegmenter::Languages::Deutsch::BetweenPunctuation.new(text: txt).replace
+        end
+
+        def replace_numbers(txt)
+          PragmaticSegmenter::Languages::Deutsch::Number.new(text: txt).replace
+        end
+
+        def replace_abbreviations(txt)
+          PragmaticSegmenter::Languages::Deutsch::AbbreviationReplacer.new(text: txt).replace
+        end
+      end
+
+      class Number < PragmaticSegmenter::Number
+        # Rubular: http://rubular.com/r/hZxoyQwKT1
+        NumberPeriodSpaceRule = Rule.new(/(?<=\s[0-9]|\s([1-9][0-9]))\.(?=\s)/, '∯')
+
+        # Rubular: http://rubular.com/r/ityNMwdghj
+        NegativeNumberPeriodSpaceRule = Rule.new(/(?<=-[0-9]|-([1-9][0-9]))\.(?=\s)/, '∯')
+
+        def replace
+          super
+          @formatted_text.apply(NumberPeriodSpaceRule).apply(NegativeNumberPeriodSpaceRule)
+        end
+      end
+
+      class SingleLetterAbbreviation < PragmaticSegmenter::SingleLetterAbbreviation
+        # Rubular: http://rubular.com/r/B4X33QKIL8
+        SingleLowerCaseLetterRule = Rule.new(/(?<=\s[a-z])\.(?=\s)/, '∯')
+
+        # Rubular: http://rubular.com/r/iUNSkCuso0
+        SingleLowerCaseLetterAtStartOfLineRule = Rule.new(/(?<=^[a-z])\.(?=\s)/, '∯')
+
+        def replace
+          super
+          @formatted_text = replace_single_lowercase_letter(@formatted_text)
+          replace_single_lowercase_letter_sol(@formatted_text)
+        end
+
+        private
+
+        def replace_single_lowercase_letter_sol(txt)
+          txt.apply(SingleLowerCaseLetterRule)
+        end
+
+        def replace_single_lowercase_letter(txt)
+          txt.apply(SingleLowerCaseLetterAtStartOfLineRule)
+        end
+      end
+
+      class Abbreviation < PragmaticSegmenter::Abbreviation
+        ABBREVIATIONS = ['Ä', 'ä', 'adj', 'adm', 'adv', 'art', 'asst', 'b.a', 'b.s', 'bart', 'bldg', 'brig', 'bros', 'bse', 'buchst', 'bzgl', 'bzw', 'c.-à-d', 'ca', 'capt', 'chr', 'cmdr', 'co', 'col', 'comdr', 'con', 'corp', 'cpl', 'd.h', 'd.j', 'dergl', 'dgl', 'dkr', 'dr ', 'ens', 'etc', 'ev ', 'evtl', 'ff', 'g.g.a', 'g.u', 'gen', 'ggf', 'gov', 'hon', 'hosp', 'i.f', 'i.h.v', 'ii', 'iii', 'insp', 'iv', 'ix', 'jun', 'k.o', 'kath ', 'lfd', 'lt', 'ltd', 'm.e', 'maj', 'med', 'messrs', 'mio', 'mlle', 'mm', 'mme', 'mr', 'mrd', 'mrs', 'ms', 'msgr', 'mwst', 'no', 'nos', 'nr', 'o.ä', 'op', 'ord', 'pfc', 'ph', 'pp', 'prof', 'pvt', 'rep', 'reps', 'res', 'rev', 'rt', 's.p.a', 'sa', 'sen', 'sens', 'sfc', 'sgt', 'sog', 'sogen', 'spp', 'sr', 'st', 'std', 'str  ', 'supt', 'surg', 'u.a  ', 'u.e', 'u.s.w', 'u.u', 'u.ä', 'usf', 'usw', 'v', 'vgl', 'vi', 'vii', 'viii', 'vs', 'x', 'xi', 'xii', 'xiii', 'xiv', 'xix', 'xv', 'xvi', 'xvii', 'xviii', 'xx', 'z.b ', 'z.t ', 'z.z', 'z.zt', 'zt', 'zzt']
+        NUMBER_ABBREVIATIONS = ['art', 'ca', 'no', 'nos', 'nr', 'pp']
+
+        def all
+          ABBREVIATIONS
+        end
+
+        def prepositive
+          []
+        end
+
+        def number
+          NUMBER_ABBREVIATIONS
+        end
+      end
+
+      class AbbreviationReplacer  < PragmaticSegmenter::AbbreviationReplacer
+        def replace
+          @reformatted_text = text.apply(PossessiveAbbreviationRule)
+          @reformatted_text = PragmaticSegmenter::Languages::Deutsch::SingleLetterAbbreviation.new(text: @reformatted_text).replace
+          @reformatted_text = search_for_abbreviations_in_string(@reformatted_text, abbreviations)
+          @reformatted_text = replace_multi_period_abbreviations(@reformatted_text)
+          @reformatted_text = @reformatted_text.apply(AmPmRules::All)
+          replace_abbreviation_as_sentence_boundary(@reformatted_text)
+        end
+
+        private
+
+        def scan_for_replacements(txt, am, index, character_array, abbr)
+          replace_abbr(txt, am)
+        end
+
+        def replace_abbr(txt, abbr)
+          txt.gsub(/(?<=#{abbr})\.(?=\s)/, '∯')
+        end
+
+        def abbreviations
+          PragmaticSegmenter::Languages::Deutsch::Abbreviation.new
+        end
+      end
+
+      class BetweenPunctuation < PragmaticSegmenter::BetweenPunctuation
+        # Rubular: http://rubular.com/r/OdcXBsub0w
+        BETWEEN_UNCONVENTIONAL_DOUBLE_QUOTE_DE_REGEX = /,,(?>[^“\\]+|\\{2}|\\.)*“/
+
+        # Rubular: http://rubular.com/r/2UskIupGgP
+        SPLIT_DOUBLE_QUOTES_DE_REGEX = /\A„(?>[^“\\]+|\\{2}|\\.)*“/
+
+        # Rubular: http://rubular.com/r/TkZomF9tTM
+        BETWEEN_DOUBLE_QUOTES_DE_REGEX = /„(?>[^“\\]+|\\{2}|\\.)*“/
+        private
+
+        def sub_punctuation_between_double_quotes(txt)
+          btwn_dbl_quote = sub_punctuation_between_double_quotes_de(txt)
+          PragmaticSegmenter::PunctuationReplacer.new(
+            matches_array: btwn_dbl_quote,
+            text: txt
+          ).replace
+        end
+
+        def sub_punctuation_between_double_quotes_de(txt)
+          if txt.include?('„')
+            btwn_dbl_quote = txt.scan(BETWEEN_DOUBLE_QUOTES_DE_REGEX)
+            txt.scan(SPLIT_DOUBLE_QUOTES_DE_REGEX).each do |q|
+              btwn_dbl_quote << q
+            end
+          elsif txt.include?(',,')
+            btwn_dbl_quote = txt.scan(BETWEEN_UNCONVENTIONAL_DOUBLE_QUOTE_DE_REGEX)
+          end
+          btwn_dbl_quote
+        end
+      end
+    end
+  end
+end

data/lib/pragmatic_segmenter/languages/english.rb

@@ -0,0 +1,44 @@
+module PragmaticSegmenter
+  module Languages
+    class English
+      class Abbreviation < PragmaticSegmenter::Abbreviation
+        ABBREVIATIONS = ['adj', 'adm', 'adv', 'al', 'ala', 'alta', 'apr', 'arc', 'ariz', 'ark', 'art', 'assn', 'asst', 'attys', 'aug', 'ave', 'bart', 'bld', 'bldg', 'blvd', 'brig', 'bros', 'cal', 'calif', 'capt', 'cl', 'cmdr', 'co', 'col', 'colo', 'comdr', 'con', 'conn', 'corp', 'cpl', 'cres', 'ct', 'd.phil', 'dak', 'dec', 'del', 'dept', 'det', 'dist', 'dr', 'dr.phil', 'dr.philos', 'drs', 'e.g', 'ens', 'esp', 'esq', 'etc', 'exp', 'expy', 'ext', 'feb', 'fed', 'fla', 'ft', 'fwy', 'fy', 'ga', 'gen', 'gov', 'hon', 'hosp', 'hr', 'hway', 'hwy', 'i.e', 'ia', 'id', 'ida', 'ill', 'inc', 'ind', 'ing', 'insp', 'is', 'jan', 'jr', 'jul', 'jun', 'kan', 'kans', 'ken', 'ky', 'la', 'lt', 'ltd', 'maj', 'man', 'mar', 'mass', 'may', 'md', 'me', 'messrs', 'mex', 'mfg', 'mich', 'min', 'minn', 'miss', 'mlle', 'mm', 'mme', 'mo', 'mont', 'mr', 'mrs', 'ms', 'msgr', 'mssrs', 'mt', 'mtn', 'neb', 'nebr', 'nev', 'no', 'nos', 'nov', 'nr', 'oct', 'ok', 'okla', 'ont', 'op', 'ord', 'ore', 'p', 'pa', 'pd', 'pde', 'penn', 'penna', 'pfc', 'ph', 'ph.d', 'pl', 'plz', 'pp', 'prof', 'pvt', 'que', 'rd', 'ref', 'rep', 'reps', 'res', 'rev', 'rt', 'sask', 'sen', 'sens', 'sep', 'sept', 'sfc', 'sgt', 'sr', 'st', 'supt', 'surg', 'tce', 'tenn', 'tex', 'univ', 'usafa', 'u.s', 'ut', 'va', 'v', 'ver', 'vs', 'vt', 'wash', 'wis', 'wisc', 'wy', 'wyo', 'yuk']
+        PREPOSITIVE_ABBREVIATIONS = ['adm', 'attys', 'brig', 'capt', 'cmdr', 'col', 'cpl', 'det', 'dr', 'gen', 'gov', 'ing', 'lt', 'maj', 'mr', 'mrs', 'ms', 'mt', 'messrs', 'mssrs', 'prof', 'rep', 'reps', 'rev', 'sen', 'sens', 'sgt', 'st', 'supt', 'v', 'vs']
+        NUMBER_ABBREVIATIONS = ['art', 'ext', 'no', 'nos', 'p', 'pp']
+
+        def all
+          ABBREVIATIONS
+        end
+
+        def prepositive
+          PREPOSITIVE_ABBREVIATIONS
+        end
+
+        def number
+          NUMBER_ABBREVIATIONS
+        end
+      end
+
+      class Cleaner < PragmaticSegmenter::Cleaner
+        def clean
+          super
+          clean_quotations(@clean_text)
+        end
+
+        private
+
+        def clean_quotations(txt)
+          txt.gsub(/`/, "'")
+        end
+      end
+
+      class AbbreviationReplacer  < PragmaticSegmenter::AbbreviationReplacer
+        private
+
+        def abbreviations
+          PragmaticSegmenter::Languages::English::Abbreviation.new
+        end
+      end
+    end
+  end
+end

data/lib/pragmatic_segmenter/languages/french.rb

@@ -0,0 +1,29 @@
+module PragmaticSegmenter
+  module Languages
+    class French
+      class Abbreviation < PragmaticSegmenter::Abbreviation
+        ABBREVIATIONS = ['a.c.n', 'a.m', 'al', 'ann', 'apr', 'art', 'auj', 'av', 'b.p', 'boul', 'c.-à-d', 'c.n', 'c.n.s', 'c.p.i', 'c.q.f.d', 'c.s', 'ca', 'cf', 'ch.-l', 'chap', 'co', 'co', 'contr', 'dir', 'e.g', 'e.v', 'env', 'etc', 'ex', 'fasc', 'fig', 'fr', 'fém', 'hab', 'i.e', 'ibid', 'id', 'inf', 'l.d', 'lib', 'll.aa', 'll.aa.ii', 'll.aa.rr', 'll.aa.ss', 'll.ee', 'll.mm', 'll.mm.ii.rr', 'loc.cit', 'ltd', 'ltd', 'masc', 'mm', 'ms', 'n.b', 'n.d', 'n.d.a', 'n.d.l.r', 'n.d.t', 'n.p.a.i', 'n.s', 'n/réf', 'nn.ss', 'p.c.c', 'p.ex', 'p.j', 'p.s', 'pl', 'pp', 'r.-v', 'r.a.s', 'r.i.p', 'r.p', 's.a', 's.a.i', 's.a.r', 's.a.s', 's.e', 's.m', 's.m.i.r', 's.s', 'sec', 'sect', 'sing', 'sq', 'sqq', 'ss', 'suiv', 'sup', 'suppl', 't.s.v.p', 'tél', 'vb', 'vol', 'vs', 'x.o', 'z.i', 'éd']
+
+        def all
+          ABBREVIATIONS
+        end
+
+        def prepositive
+          []
+        end
+
+        def number
+          []
+        end
+      end
+
+      class AbbreviationReplacer  < PragmaticSegmenter::AbbreviationReplacer
+        private
+
+        def abbreviations
+          PragmaticSegmenter::Languages::French::Abbreviation.new
+        end
+      end
+    end
+  end
+end

data/lib/pragmatic_segmenter/languages/greek.rb

@@ -0,0 +1,29 @@
+module PragmaticSegmenter
+  module Languages
+    class Greek
+      class Process < PragmaticSegmenter::Process
+        private
+
+        def sentence_boundary_punctuation(txt)
+          PragmaticSegmenter::Languages::Greek::SentenceBoundaryPunctuation.new(text: txt).split
+        end
+      end
+
+      class SentenceBoundaryPunctuation < PragmaticSegmenter::SentenceBoundaryPunctuation
+        SENTENCE_BOUNDARY = /.*?[\.;!\?]|.*?$/
+
+        def split
+          text.scan(SENTENCE_BOUNDARY)
+        end
+      end
+
+      class Punctuation < PragmaticSegmenter::Punctuation
+        PUNCT = ['.', '!', ';', '?']
+
+        def punct
+          PUNCT
+        end
+      end
+    end
+  end
+end

data/lib/pragmatic_segmenter/languages/hindi.rb

@@ -0,0 +1,33 @@
+module PragmaticSegmenter
+  module Languages
+    class Hindi
+      class Process < PragmaticSegmenter::Process
+        private
+
+        def sentence_boundary_punctuation(txt)
+          PragmaticSegmenter::Languages::Hindi::SentenceBoundaryPunctuation.new(text: txt).split
+        end
+
+        def punctuation_array
+          PragmaticSegmenter::Languages::Hindi::Punctuation.new.punct
+        end
+      end
+
+      class SentenceBoundaryPunctuation < PragmaticSegmenter::SentenceBoundaryPunctuation
+        SENTENCE_BOUNDARY = /.*?[।\|!\?]|.*?$/
+
+        def split
+          text.scan(SENTENCE_BOUNDARY)
+        end
+      end
+
+      class Punctuation < PragmaticSegmenter::Punctuation
+        PUNCT = ['।', '|', '.', '!', '?']
+
+        def punct
+          PUNCT
+        end
+      end
+    end
+  end
+end