pragmatic_segmenter 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e72b3b2f49ed11b535428359d9b226b3de207fa1
-  data.tar.gz: 97be84a29ae5dab929c3ad0cff6d14789729b620
+  metadata.gz: 7e52d1869830dfba91d5e5a00f15d3529081691e
+  data.tar.gz: 5754723c0ba657a31e3471d785b034e3c1814e33
 SHA512:
-  metadata.gz: 92a96798cf0de8d50f89239edd860a3c97ba4b20a07ba28bd9535b0fd0292297bbe60ae20ad297265fbcb7af6e8578156e271f86258a56be745b57c563c59dba
-  data.tar.gz: 82e8dfbfa66fedf6b510d52e16844eee5b2defcc278d809ad743764f010affb2742d37374a41c3e9638c73e074311d4c6708fa984fba162eb8b2f1f73eb27f1f
+  metadata.gz: d3a4700ba0369b60d36f633c4e042a5113197956ad02dff64243c70445bc445435af17689f456327753cead2d9d1cee00db548e2e0dad337341f110194522d2c
+  data.tar.gz: f8a28d6582e846ad2ffff21b87f4888b922cc4224faaf85e2782ca92671506dbf55fab86c0ddf06ae6363a3851f8ec9dd44f7bc9956a86f2ed2aea4bc6d69fb5

data/README.md

@@ -21,7 +21,7 @@ gem 'pragmatic_segmenter'
 ##Usage  
 
 * If no language is specified, the library will default to English.   
-* To specify a language use its two-digit [ISO 639-1 code](https://www.tm-town.com/languages).  
+* To specify a language use its two character [ISO 639-1 code](https://www.tm-town.com/languages).  
 
 ```ruby
 text = "Hello world. My name is Mr. Smith. I work for the U.S. Government and I live in the U.S. I live in New York."
@@ -60,7 +60,7 @@ Try out a [live demo](https://www.tm-town.com/natural-language-processing) of Pr
 
 ##Background
 
-According to Wikipedia, [sentence boundary disambiguation](http://en.wikipedia.org/wiki/Sentence_boundary_disambiguation) (aka sentence boundary disambiguation, sentence segmentation) is defined as:
+According to Wikipedia, [sentence boundary disambiguation](http://en.wikipedia.org/wiki/Sentence_boundary_disambiguation) (aka sentence boundary detection, sentence segmentation) is defined as:
 
 > Sentence boundary disambiguation (SBD), also known as sentence breaking, is the problem in natural language processing of deciding where sentences begin and end. Often natural language processing tools require their input to be divided into sentences for a number of reasons. However sentence boundary identification is challenging because punctuation marks are often ambiguous. For example, a period may denote an abbreviation, decimal point, an ellipsis, or an email address – not the end of a sentence. About 47% of the periods in the Wall Street Journal corpus denote abbreviations. As well, question marks and exclamation marks may appear in embedded quotations, emoticons, computer code, and slang. Languages like Japanese and Chinese have unambiguous sentence-ending markers.
 
@@ -677,6 +677,9 @@ Other tools not yet tested:
 * [Ucto: Unicode Tokenizer](http://ilk.uvt.nl/ucto/)
 * [tokenizer](http://moin.delph-in.net/WeSearch/DocumentParsing)
 * [spaCy](http://honnibal.github.io/spaCy/)
+* [GATE](https://gate.ac.uk/)
+* [University of Illinois Sentence Segmentation tool](http://cogcomp.cs.illinois.edu/page/tools_view/2)
+* [DetectorMorse](https://github.com/cslu-nlp/detectormorse)
 
 ## Speed Performance Benchmarks
 
@@ -810,6 +813,9 @@ To test the relative performance of different segmentation tools and libraries I
 **Version 0.3.1**  
 * Fix undefined method 'gsub!' for nil:NilClass issue  
 
+**Version 0.3.2**  
+* Add English abbreviations  
+
 ## Contributing
 
 If you find a text that is incorrectly segmented using this gem, please submit an issue.

data/lib/pragmatic_segmenter/abbreviation_replacer.rb

@@ -1,75 +1,51 @@
 # -*- encoding : utf-8 -*-
-require 'pragmatic_segmenter/abbreviation'
-require 'pragmatic_segmenter/single_letter_abbreviation'
 
 module PragmaticSegmenter
   # This class searches for periods within an abbreviation and
   # replaces the periods.
   class AbbreviationReplacer
-    # Rubular: http://rubular.com/r/yqa4Rit8EY
-    PossessiveAbbreviationRule = Rule.new(/\.(?='s\s)|\.(?='s$)|\.(?='s\z)/, '∯')
-
-    # Rubular: http://rubular.com/r/NEv265G2X2
-    KommanditgesellschaftRule = Rule.new(/(?<=Co)\.(?=\sKG)/, '∯')
-
-    # Rubular: http://rubular.com/r/xDkpFZ0EgH
-    MULTI_PERIOD_ABBREVIATION_REGEX = /\b[a-z](?:\.[a-z])+[.]/i
-
-    module AmPmRules
-      # Rubular: http://rubular.com/r/Vnx3m4Spc8
-      UpperCasePmRule = Rule.new(/(?<=P∯M)∯(?=\s[A-Z])/, '.')
-
-      # Rubular: http://rubular.com/r/AJMCotJVbW
-      UpperCaseAmRule = Rule.new(/(?<=A∯M)∯(?=\s[A-Z])/, '.')
-
-      # Rubular: http://rubular.com/r/13q7SnOhgA
-      LowerCasePmRule = Rule.new(/(?<=p∯m)∯(?=\s[A-Z])/, '.')
-
-      # Rubular: http://rubular.com/r/DgUDq4mLz5
-      LowerCaseAmRule = Rule.new(/(?<=a∯m)∯(?=\s[A-Z])/, '.')
-
-      All = [UpperCasePmRule, UpperCaseAmRule, LowerCasePmRule, LowerCaseAmRule]
-    end
 
     SENTENCE_STARTERS = %w(A Being Did For He How However I In It Millions More She That The There They We What When Where Who Why)
 
     attr_reader :text
-    def initialize(text:)
+    def initialize(text:, language: Languages::Common)
       @text = Text.new(text)
+      @language = language
     end
 
     def replace
-      @reformatted_text = text.apply(PossessiveAbbreviationRule)
-      @reformatted_text = text.apply(KommanditgesellschaftRule)
-      @reformatted_text = PragmaticSegmenter::SingleLetterAbbreviation.new(text: @reformatted_text).replace
-      @reformatted_text = search_for_abbreviations_in_string(@reformatted_text, abbreviations)
+      @reformatted_text = text.apply(@language::PossessiveAbbreviationRule,
+        @language::KommanditgesellschaftRule,
+        @language::SingleLetterAbbreviationRules::All)
+
+      @reformatted_text = search_for_abbreviations_in_string(@reformatted_text)
       @reformatted_text = replace_multi_period_abbreviations(@reformatted_text)
-      @reformatted_text = @reformatted_text.apply(AmPmRules::All)
+      @reformatted_text = @reformatted_text.apply(@language::AmPmRules::All)
       replace_abbreviation_as_sentence_boundary(@reformatted_text)
     end
 
     private
 
-    def search_for_abbreviations_in_string(txt, abbr)
+    def search_for_abbreviations_in_string(txt)
       original = txt.dup
       downcased = txt.downcase
-      abbr.all.each do |a|
+      @language::Abbreviation::ABBREVIATIONS.each do |a|
         next unless downcased.include?(a.strip)
         abbrev_match = original.scan(/(?:^|\s|\r|\n)#{Regexp.escape(a.strip)}/i)
         next if abbrev_match.empty?
         next_word_start = /(?<=#{Regexp.escape(a.strip)} ).{1}/
         character_array = @text.scan(next_word_start)
         abbrev_match.each_with_index do |am, index|
-          txt = scan_for_replacements(txt, am, index, character_array, abbr)
+          txt = scan_for_replacements(txt, am, index, character_array)
         end
       end
       txt
     end
 
-    def scan_for_replacements(txt, am, index, character_array, abbr)
+    def scan_for_replacements(txt, am, index, character_array)
       character = character_array[index]
-      prepositive = abbr.prepositive
-      number_abbr = abbr.number
+      prepositive = @language::Abbreviation::PREPOSITIVE_ABBREVIATIONS
+      number_abbr = @language::Abbreviation::NUMBER_ABBREVIATIONS
       upper = /[[:upper:]]/.match(character.to_s)
       if upper.nil? || prepositive.include?(am.downcase.strip)
         if prepositive.include?(am.downcase.strip)
@@ -83,10 +59,6 @@ module PragmaticSegmenter
       txt
     end
 
-    def abbreviations
-      @abbr ||= PragmaticSegmenter::Abbreviation.new
-    end
-
     def replace_abbreviation_as_sentence_boundary(txt)
       # As we are being conservative and keeping ambiguous
       # sentence boundaries as one sentence instead of
@@ -120,7 +92,7 @@ module PragmaticSegmenter
     end
 
     def replace_multi_period_abbreviations(txt)
-      mpa = txt.scan(MULTI_PERIOD_ABBREVIATION_REGEX)
+      mpa = txt.scan(@language::MULTI_PERIOD_ABBREVIATION_REGEX)
       return txt if mpa.empty?
       mpa.each do |r|
         txt = txt.gsub(/#{Regexp.escape(r)}/, "#{r.gsub!('.', '∯')}")
@@ -128,13 +100,6 @@ module PragmaticSegmenter
       txt
     end
 
-    def replace_period_in_am_pm(txt)
-      txt.gsub(UPPERCASE_PM_REGEX, '.')
-         .gsub(UPPERCASE_AM_REGEX, '.')
-         .gsub(LOWERCASE_PM_REGEX, '.')
-         .gsub(LOWERCASE_AM_REGEX, '.')
-    end
-
     def replace_pre_number_abbr(txt, abbr)
       txt.gsub(/(?<=\s#{abbr.strip})\.(?=\s\d)|(?<=^#{abbr.strip})\.(?=\s\d)/, '∯')
          .gsub(/(?<=\s#{abbr.strip})\.(?=\s+\()|(?<=^#{abbr.strip})\.(?=\s+\()/, '∯')
@@ -152,7 +117,7 @@ module PragmaticSegmenter
     end
 
     def replace_possessive_abbreviations(txt)
-      txt.gsub(POSSESSIVE_ABBREVIATION_REGEX, '∯')
+      txt.gsub(@language::POSSESSIVE_ABBREVIATION_REGEX, '∯')
     end
   end
 end

data/lib/pragmatic_segmenter/cleaner.rb

@@ -1,87 +1,16 @@
 # -*- encoding : utf-8 -*-
 
 module PragmaticSegmenter
-  module Rules
-    module HtmlRules
-      # Rubular: http://rubular.com/r/ENrVFMdJ8v
-      HTMLTagRule = Rule.new(/<\/?[^>]*>/, '')
-
-      # Rubular: http://rubular.com/r/XZVqMPJhea
-      EscapedHTMLTagRule = Rule.new(/&lt;\/?[^gt;]*gt;/, '')
-
-      All = [HTMLTagRule, EscapedHTMLTagRule]
-    end
-  end
-
   # This is an opinionated class that removes errant newlines,
   # xhtml, inline formatting, etc.
   class Cleaner
     include Rules
-    URL_EMAIL_KEYWORDS = ['@', 'http', '.com', 'net', 'www', '//']
-
-    # Rubular: http://rubular.com/r/6dt98uI76u
-    NO_SPACE_BETWEEN_SENTENCES_REGEX = /(?<=[a-z])\.(?=[A-Z])/
-
-    # Rubular: http://rubular.com/r/l6KN6rH5XE
-    NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX = /(?<=\d)\.(?=[A-Z])/
-
-    # Rubular: http://rubular.com/r/V57WnM9Zut
-    NewLineInMiddleOfWordRule = Rule.new(/\n(?=[a-zA-Z]{1,2}\n)/, '')
-
-    # Rubular: http://rubular.com/r/3GiRiP2IbD
-    NEWLINE_IN_MIDDLE_OF_SENTENCE_REGEX = /(?<=\s)\n(?=([a-z]|\())/
-
-    # Rubular: http://rubular.com/r/UZAVcwqck8
-    PDF_NewLineInMiddleOfSentenceRule = Rule.new(/(?<=[^\n]\s)\n(?=\S)/, '')
-
-    # Rubular: http://rubular.com/r/eaNwGavmdo
-    PDF_NewLineInMiddleOfSentenceNoSpacesRule = Rule.new(/\n(?=[a-z])/, ' ')
-
-    # Rubular: http://rubular.com/r/bAJrhyLNeZ
-    InlineFormattingRule = Rule.new(/\{b\^&gt;\d*&lt;b\^\}|\{b\^>\d*<b\^\}/, '')
-
-    # Rubular: http://rubular.com/r/dMxp5MixFS
-    DoubleNewLineWithSpaceRule = Rule.new(/\n \n/, "\r")
-
-    # Rubular: http://rubular.com/r/H6HOJeA8bq
-    DoubleNewLineRule = Rule.new(/\n\n/, "\r")
-
-    # Rubular: http://rubular.com/r/Gn18aAnLdZ
-    NewLineFollowedByBulletRule = Rule.new(/\n(?=•)/, "\r")
-
-    # Rubular: http://rubular.com/r/FseyMiiYFT
-    NewLineFollowedByPeriodRule = Rule.new(/\n(?=\.(\s|\n))/, '')
-
-    # Rubular: http://rubular.com/r/8mc1ArOIGy
-    TableOfContentsRule = Rule.new(/\.{5,}\s*\d+-*\d*/, "\r")
-
-    # Rubular: http://rubular.com/r/DwNSuZrNtk
-    ConsecutivePeriodsRule = Rule.new(/\.{5,}/, ' ')
-
-    # Rubular: http://rubular.com/r/IQ4TPfsbd8
-    ConsecutiveForwardSlashRule = Rule.new(/\/{3}/, '')
-
-    # Rubular: http://rubular.com/r/6dt98uI76u
-    NoSpaceBetweenSentencesRule = Rule.new(NO_SPACE_BETWEEN_SENTENCES_REGEX, '. ')
-
-    # Rubular: http://rubular.com/r/l6KN6rH5XE
-    NoSpaceBetweenSentencesDigitRule = Rule.new(NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX, '. ')
-
-    EscapedCarriageReturnRule = Rule.new(/\\r/, "\r")
-    TypoEscapedCarriageReturnRule = Rule.new(/\\\ r/, "\r")
-
-    EscapedNewLineRule = Rule.new(/\\n/, "\n")
-    TypoEscapedNewLineRule = Rule.new(/\\\ n/, "\n")
-
-    ReplaceNewlineWithCarriageReturnRule = Rule.new(/\n/, "\r")
-
-    QuotationsFirstRule = Rule.new(/''/, '"')
-    QuotationsSecondRule = Rule.new(/``/, '"')
 
     attr_reader :text, :doc_type
-    def initialize(text:, **args)
+    def initialize(text:, doc_type: nil, language: Languages::Common, **args)
       @text = Text.new(text.dup)
-      @doc_type = args[:doc_type]
+      @doc_type = doc_type
+      @language = language
     end
 
     # Clean text of unwanted formatting
@@ -94,7 +23,7 @@ module PragmaticSegmenter
     # Arguments:
     #    text:       (String)  *required
     #    language:   (String)  *optional
-    #                (two-digit ISO 639-1 code e.g. 'en')
+    #                (two character ISO 639-1 code e.g. 'en')
     #    doc_type:   (String)  *optional
     #                (e.g. 'pdf')
 
@@ -104,7 +33,7 @@ module PragmaticSegmenter
       replace_double_newlines(@clean_text)
       replace_newlines(@clean_text)
       replace_escaped_newlines(@clean_text)
-      @clean_text.apply(HtmlRules::All)
+      @clean_text.apply(HTMLRules::All)
       replace_punctuation_in_brackets(@clean_text)
       @clean_text.apply(InlineFormattingRule)
       clean_quotations(@clean_text)
@@ -141,10 +70,6 @@ module PragmaticSegmenter
       end
     end
 
-    def abbreviations
-      @abbr ||= PragmaticSegmenter::Abbreviation.new.all
-    end
-
     def remove_all_newlines(txt)
       clean_text = remove_newline_in_middle_of_sentence(txt)
       remove_newline_in_middle_of_word(clean_text)
@@ -161,50 +86,44 @@ module PragmaticSegmenter
     end
 
     def remove_newline_in_middle_of_word(txt)
-      txt.apply(NewLineInMiddleOfWordRule)
+      txt.apply NewLineInMiddleOfWordRule
     end
 
     def replace_escaped_newlines(txt)
-      txt.apply(EscapedNewLineRule).
-          apply(EscapedCarriageReturnRule).
-          apply(TypoEscapedNewLineRule).
-          apply(TypoEscapedCarriageReturnRule)
+      txt.apply EscapedNewLineRule, EscapedCarriageReturnRule,
+        TypoEscapedNewLineRule, TypoEscapedCarriageReturnRule
     end
 
     def replace_double_newlines(txt)
-      txt.apply(DoubleNewLineWithSpaceRule).
-          apply(DoubleNewLineRule)
+      txt.apply DoubleNewLineWithSpaceRule, DoubleNewLineRule
     end
 
     def replace_newlines(txt)
       if doc_type.eql?('pdf')
         remove_pdf_line_breaks(txt)
       else
-        txt.apply(NewLineFollowedByPeriodRule).
-            apply(ReplaceNewlineWithCarriageReturnRule)
+        txt.apply NewLineFollowedByPeriodRule,
+          ReplaceNewlineWithCarriageReturnRule
       end
     end
 
     def remove_pdf_line_breaks(txt)
-      txt.apply(NewLineFollowedByBulletRule).
-          apply(PDF_NewLineInMiddleOfSentenceRule).
-          apply(PDF_NewLineInMiddleOfSentenceNoSpacesRule)
+      txt.apply NewLineFollowedByBulletRule,
+        PDF_NewLineInMiddleOfSentenceRule,
+        PDF_NewLineInMiddleOfSentenceNoSpacesRule
     end
 
     def clean_quotations(txt)
-      txt.apply(QuotationsFirstRule).
-          apply(QuotationsSecondRule)
+      txt.apply QuotationsFirstRule, QuotationsSecondRule
     end
 
     def clean_table_of_contents(txt)
-      txt.apply(TableOfContentsRule).
-          apply(ConsecutivePeriodsRule).
-          apply(ConsecutiveForwardSlashRule)
+      txt.apply TableOfContentsRule, ConsecutivePeriodsRule,
+        ConsecutiveForwardSlashRule
     end
 
     def clean_consecutive_characters(txt)
-      txt.apply(ConsecutivePeriodsRule).
-          apply(ConsecutiveForwardSlashRule)
+      txt.apply ConsecutivePeriodsRule, ConsecutiveForwardSlashRule
     end
   end
 end

data/lib/pragmatic_segmenter/languages.rb

@@ -0,0 +1,62 @@
+require 'pragmatic_segmenter/types'
+require 'pragmatic_segmenter/process'
+require 'pragmatic_segmenter/cleaner'
+require 'pragmatic_segmenter/rules'
+
+require 'pragmatic_segmenter/languages/common'
+
+require 'pragmatic_segmenter/languages/english'
+require 'pragmatic_segmenter/languages/deutsch'
+require 'pragmatic_segmenter/languages/hindi'
+require 'pragmatic_segmenter/languages/persian'
+require 'pragmatic_segmenter/languages/amharic'
+require 'pragmatic_segmenter/languages/arabic'
+require 'pragmatic_segmenter/languages/greek'
+require 'pragmatic_segmenter/languages/armenian'
+require 'pragmatic_segmenter/languages/burmese'
+require 'pragmatic_segmenter/languages/urdu'
+require 'pragmatic_segmenter/languages/french'
+require 'pragmatic_segmenter/languages/italian'
+require 'pragmatic_segmenter/languages/spanish'
+require 'pragmatic_segmenter/languages/russian'
+require 'pragmatic_segmenter/languages/japanese'
+require 'pragmatic_segmenter/languages/dutch'
+require 'pragmatic_segmenter/languages/polish'
+require 'pragmatic_segmenter/languages/chinese'
+
+module PragmaticSegmenter
+  module Languages
+    LANGUAGE_CODES = {
+      'en' => 'English',
+      'de' => 'Deutsch',
+      'es' => 'Spanish',
+      'fr' => 'French',
+      'it' => 'Italian',
+      'ja' => 'Japanese',
+      'el' => 'Greek',
+      'ru' => 'Russian',
+      'ar' => 'Arabic',
+      'am' => 'Amharic',
+      'hi' => 'Hindi',
+      'hy' => 'Armenian',
+      'fa' => 'Persian',
+      'my' => 'Burmese',
+      'ur' => 'Urdu',
+      'nl' => 'Dutch',
+      'pl' => 'Polish',
+      'zh' => 'Chinese',
+    }
+
+    def process_class
+      language_module::Process
+    end
+
+    def cleaner_class
+      language_module::Cleaner
+    end
+
+    def language_module
+      Object.const_get("PragmaticSegmenter::Languages::#{LANGUAGE_CODES[language] || 'Common'}")
+    end
+  end
+end

data/lib/pragmatic_segmenter/languages/amharic.rb

@@ -1,36 +1,10 @@
 module PragmaticSegmenter
   module Languages
-    class Amharic
-      class Process < PragmaticSegmenter::Process
-        private
+    module Amharic
+      include Languages::Common
 
-        def sentence_boundary_punctuation(txt)
-          PragmaticSegmenter::Languages::Amharic::SentenceBoundaryPunctuation.new(text: txt).split
-        end
-
-        def punctuation_array
-          PragmaticSegmenter::Languages::Amharic::Punctuation.new.punct
-        end
-      end
-
-      class Cleaner < PragmaticSegmenter::Cleaner
-      end
-
-      class SentenceBoundaryPunctuation < PragmaticSegmenter::SentenceBoundaryPunctuation
-        SENTENCE_BOUNDARY = /.*?[፧።!\?]|.*?$/
-
-        def split
-          text.scan(SENTENCE_BOUNDARY)
-        end
-      end
-
-      class Punctuation < PragmaticSegmenter::Punctuation
-        PUNCT = ['።', '፧', '?', '!']
-
-        def punct
-          PUNCT
-        end
-      end
+      SENTENCE_BOUNDARY_REGEX = /.*?[፧።!\?]|.*?$/
+      Punctuations = ['።', '፧', '?', '!']
     end
   end
 end