pragmatic_segmenter 0.0.1

data/Rakefile

@@ -0,0 +1,4 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)

data/lib/pragmatic_segmenter.rb

@@ -0,0 +1,2 @@
+require "pragmatic_segmenter/version"
+require "pragmatic_segmenter/segmenter"

data/lib/pragmatic_segmenter/abbreviation.rb

@@ -0,0 +1,22 @@
+# -*- encoding : utf-8 -*-
+
+module PragmaticSegmenter
+  # Defines the abbreviations for each language (if available)
+  class Abbreviation
+    ABBREVIATIONS = ['adj', 'adm', 'adv', 'al', 'ala', 'alta', 'apr', 'arc', 'ariz', 'ark', 'art', 'assn', 'asst', 'attys', 'aug', 'ave', 'bart', 'bld', 'bldg', 'blvd', 'brig', 'bros', 'cal', 'calif', 'capt', 'cl', 'cmdr', 'co', 'col', 'colo', 'comdr', 'con', 'conn', 'corp', 'cpl', 'cres', 'ct', 'd.phil', 'dak', 'dec', 'del', 'dept', 'det', 'dist', 'dr', 'dr.phil', 'dr.philos', 'drs', 'e.g', 'ens', 'esp', 'esq', 'etc', 'exp', 'expy', 'ext', 'feb', 'fed', 'fla', 'ft', 'fwy', 'fy', 'ga', 'gen', 'gov', 'hon', 'hosp', 'hr', 'hway', 'hwy', 'i.e', 'ia', 'id', 'ida', 'ill', 'inc', 'ind', 'ing', 'insp', 'is', 'jan', 'jr', 'jul', 'jun', 'kan', 'kans', 'ken', 'ky', 'la', 'lt', 'ltd', 'maj', 'man', 'mar', 'mass', 'may', 'md', 'me', 'messrs', 'mex', 'mfg', 'mich', 'min', 'minn', 'miss', 'mlle', 'mm', 'mme', 'mo', 'mont', 'mr', 'mrs', 'ms', 'msgr', 'mssrs', 'mt', 'mtn', 'neb', 'nebr', 'nev', 'no', 'nos', 'nov', 'nr', 'oct', 'ok', 'okla', 'ont', 'op', 'ord', 'ore', 'p', 'pa', 'pd', 'pde', 'penn', 'penna', 'pfc', 'ph', 'ph.d', 'pl', 'plz', 'pp', 'prof', 'pvt', 'que', 'rd', 'ref', 'rep', 'reps', 'res', 'rev', 'rt', 'sask', 'sen', 'sens', 'sep', 'sept', 'sfc', 'sgt', 'sr', 'st', 'supt', 'surg', 'tce', 'tenn', 'tex', 'univ', 'usafa', 'u.s', 'ut', 'va', 'v', 'ver', 'vs', 'vt', 'wash', 'wis', 'wisc', 'wy', 'wyo', 'yuk']
+    PREPOSITIVE_ABBREVIATIONS = ['adm', 'attys', 'brig', 'capt', 'cmdr', 'col', 'cpl', 'det', 'dr', 'gen', 'gov', 'ing', 'lt', 'maj', 'mr', 'mrs', 'ms', 'mt', 'messrs', 'mssrs', 'prof', 'rep', 'reps', 'rev', 'sen', 'sens', 'sgt', 'st', 'supt', 'v', 'vs']
+    NUMBER_ABBREVIATIONS = ['art', 'ext', 'no', 'nos', 'p', 'pp']
+
+    def all
+      ABBREVIATIONS
+    end
+
+    def prepositive
+      PREPOSITIVE_ABBREVIATIONS
+    end
+
+    def number
+      NUMBER_ABBREVIATIONS
+    end
+  end
+end

data/lib/pragmatic_segmenter/abbreviation_replacer.rb

@@ -0,0 +1,149 @@
+# -*- encoding : utf-8 -*-
+require 'pragmatic_segmenter/abbreviation'
+require 'pragmatic_segmenter/single_letter_abbreviation'
+
+module PragmaticSegmenter
+  # This class searches for periods within an abbreviation and
+  # replaces the periods.
+  class AbbreviationReplacer
+    # Rubular: http://rubular.com/r/yqa4Rit8EY
+    PossessiveAbbreviationRule = Rule.new(/\.(?='s\s)|\.(?='s$)|\.(?='s\z)/, '∯')
+
+    # Rubular: http://rubular.com/r/xDkpFZ0EgH
+    MULTI_PERIOD_ABBREVIATION_REGEX = /\b[a-z](?:\.[a-z])+[.]/i
+
+    module AmPmRules
+      # Rubular: http://rubular.com/r/Vnx3m4Spc8
+      UpperCasePmRule = Rule.new(/(?<=P∯M)∯(?=\s[A-Z])/, '.')
+
+      # Rubular: http://rubular.com/r/AJMCotJVbW
+      UpperCaseAmRule = Rule.new(/(?<=A∯M)∯(?=\s[A-Z])/, '.')
+
+      # Rubular: http://rubular.com/r/13q7SnOhgA
+      LowerCasePmRule = Rule.new(/(?<=p∯m)∯(?=\s[A-Z])/, '.')
+
+      # Rubular: http://rubular.com/r/DgUDq4mLz5
+      LowerCaseAmRule = Rule.new(/(?<=a∯m)∯(?=\s[A-Z])/, '.')
+
+      All = [UpperCasePmRule, UpperCaseAmRule, LowerCasePmRule, LowerCaseAmRule]
+    end
+
+    SENTENCE_STARTERS = %w(A Being Did For He How However I In Millions More She That The There They We What When Where Who Why)
+
+    attr_reader :text
+    def initialize(text:)
+      @text = Text.new(text)
+    end
+
+    def replace
+      @reformatted_text = text.apply(PossessiveAbbreviationRule)
+      @reformatted_text = PragmaticSegmenter::SingleLetterAbbreviation.new(text: @reformatted_text).replace
+      @reformatted_text = search_for_abbreviations_in_string(@reformatted_text, abbreviations)
+      @reformatted_text = replace_multi_period_abbreviations(@reformatted_text)
+      @reformatted_text = @reformatted_text.apply(AmPmRules::All)
+      replace_abbreviation_as_sentence_boundary(@reformatted_text)
+    end
+
+    private
+
+    def search_for_abbreviations_in_string(txt, abbr)
+      original = txt.dup
+      downcased = txt.downcase
+      abbr.all.each do |a|
+        next unless downcased.include?(a.strip)
+        abbrev_match = original.scan(/(?:^|\s|\r|\n)#{Regexp.escape(a.strip)}/i)
+        next if abbrev_match.empty?
+        next_word_start = /(?<=#{Regexp.escape(a.strip)} ).{1}/
+        character_array = @text.scan(next_word_start)
+        abbrev_match.each_with_index do |am, index|
+          txt = scan_for_replacements(txt, am, index, character_array, abbr)
+        end
+      end
+      txt
+    end
+
+    def scan_for_replacements(txt, am, index, character_array, abbr)
+      character = character_array[index]
+      prepositive = abbr.prepositive
+      number_abbr = abbr.number
+      upper = /[[:upper:]]/.match(character.to_s)
+      if upper.nil? || prepositive.include?(am.downcase.strip)
+        if prepositive.include?(am.downcase.strip)
+          txt = replace_prepositive_abbr(txt, am)
+        elsif number_abbr.include?(am.downcase.strip)
+          txt = replace_pre_number_abbr(txt, am)
+        else
+          txt = replace_period_of_abbr(txt, am)
+        end
+      end
+      txt
+    end
+
+    def abbreviations
+      PragmaticSegmenter::Abbreviation.new
+    end
+
+    def replace_abbreviation_as_sentence_boundary(txt)
+      # As we are being conservative and keeping ambiguous
+      # sentence boundaries as one sentence instead of
+      # splitting into two, we can split at words that
+      # we know for certain never follow these abbreviations.
+      # Some might say that the set of words that follow an
+      # abbreviation such as U.S. (i.e. U.S. Government) is smaller than
+      # the set of words that could start a sentence and
+      # never follow U.S. However, we  are being conservative
+      # and not splitting by default, so we need to look for places
+      # where we definitely can split. Obviously SENTENCE_STARTERS
+      # will never cover all cases, but as the gem is named
+      # 'Pragmatic Segmenter' we need to be pragmatic
+      # and try to cover the words that most often start a
+      # sentence but could never follow one of the abbreviations below.
+
+      SENTENCE_STARTERS.each do |word|
+        txt = txt.gsub(/U∯S∯\s#{Regexp.escape(word)}\s/, "U∯S\.\s#{Regexp.escape(word)}\s")
+              .gsub(/U\.S∯\s#{Regexp.escape(word)}\s/, "U\.S\.\s#{Regexp.escape(word)}\s")
+              .gsub(/U∯K∯\s#{Regexp.escape(word)}\s/, "U∯K\.\s#{Regexp.escape(word)}\s")
+              .gsub(/U\.K∯\s#{Regexp.escape(word)}\s/, "U\.K\.\s#{Regexp.escape(word)}\s")
+              .gsub(/E∯U∯\s#{Regexp.escape(word)}\s/, "E∯U\.\s#{Regexp.escape(word)}\s")
+              .gsub(/E\.U∯\s#{Regexp.escape(word)}\s/, "E\.U\.\s#{Regexp.escape(word)}\s")
+              .gsub(/U∯S∯A∯\s#{Regexp.escape(word)}\s/, "U∯S∯A\.\s#{Regexp.escape(word)}\s")
+              .gsub(/U\.S\.A∯\s#{Regexp.escape(word)}\s/, "U\.S\.A\.\s#{Regexp.escape(word)}\s")
+              .gsub(/I∯\s#{Regexp.escape(word)}\s/, "I\.\s#{Regexp.escape(word)}\s")
+      end
+      txt
+    end
+
+    def replace_multi_period_abbreviations(txt)
+      mpa = txt.scan(MULTI_PERIOD_ABBREVIATION_REGEX)
+      return txt if mpa.empty?
+      mpa.each do |r|
+        txt = txt.gsub(/#{Regexp.escape(r)}/, "#{r.gsub!('.', '∯')}")
+      end
+      txt
+    end
+
+    def replace_period_in_am_pm(txt)
+      txt.gsub(UPPERCASE_PM_REGEX, '.')
+        .gsub(UPPERCASE_AM_REGEX, '.')
+        .gsub(LOWERCASE_PM_REGEX, '.')
+        .gsub(LOWERCASE_AM_REGEX, '.')
+    end
+
+    def replace_pre_number_abbr(txt, abbr)
+      txt.gsub(/(?<=#{abbr.strip})\.(?=\s\d)/, '∯').gsub(/(?<=#{abbr.strip})\.(?=\s+\()/, '∯')
+    end
+
+    def replace_prepositive_abbr(txt, abbr)
+      txt.gsub(/(?<=#{abbr.strip})\.(?=\s)/, '∯')
+    end
+
+    def replace_period_of_abbr(txt, abbr)
+      txt.gsub(/(?<=#{abbr.strip})\.(?=((\.|:|\?)|(\s([a-z]|I\s|I'm|I'll|\d))))/, '∯')
+        .gsub(/(?<=#{abbr.strip})\.(?=,)/, '∯')
+    end
+
+    def replace_possessive_abbreviations(txt)
+      txt.gsub(POSSESSIVE_ABBREVIATION_REGEX, '∯')
+    end
+  end
+end

data/lib/pragmatic_segmenter/between_punctuation.rb

@@ -0,0 +1,78 @@
+# -*- encoding : utf-8 -*-
+require 'pragmatic_segmenter/punctuation_replacer'
+
+module PragmaticSegmenter
+  # This class searches for punctuation between quotes or parenthesis
+  # and replaces it
+  class BetweenPunctuation
+    # Rubular: http://rubular.com/r/2YFrKWQUYi
+    BETWEEN_SINGLE_QUOTES_REGEX = /(?<=\s)'(?:[^']|'[a-zA-Z])*'/
+
+    # Rubular: http://rubular.com/r/3Pw1QlXOjd
+    BETWEEN_DOUBLE_QUOTES_REGEX = /"(?>[^"\\]+|\\{2}|\\.)*"/
+
+    # Rubular: http://rubular.com/r/x6s4PZK8jc
+    BETWEEN_QUOTE_ARROW_REGEX = /«(?>[^»\\]+|\\{2}|\\.)*»/
+
+    # Rubular: http://rubular.com/r/JbAIpKdlSq
+    BETWEEN_QUOTE_SLANTED_REGEX = /“(?>[^”\\]+|\\{2}|\\.)*”/
+
+    # Rubular: http://rubular.com/r/6tTityPflI
+    BETWEEN_PARENS_REGEX = /\((?>[^\(\)\\]+|\\{2}|\\.)*\)/
+
+    attr_reader :text
+    def initialize(text:)
+      @text = text
+    end
+
+    def replace
+      sub_punctuation_between_quotes_and_parens(text)
+    end
+
+    private
+
+    def sub_punctuation_between_quotes_and_parens(txt)
+      sub_punctuation_between_single_quotes(txt)
+      sub_punctuation_between_double_quotes(txt)
+      sub_punctuation_between_parens(txt)
+      sub_punctuation_between_quotes_arrow(txt)
+      sub_punctuation_between_quotes_slanted(txt)
+    end
+
+    def sub_punctuation_between_parens(txt)
+      PragmaticSegmenter::PunctuationReplacer.new(
+        matches_array: txt.scan(BETWEEN_PARENS_REGEX),
+        text: txt
+      ).replace
+    end
+
+    def sub_punctuation_between_single_quotes(txt)
+      PragmaticSegmenter::PunctuationReplacer.new(
+        matches_array: txt.scan(BETWEEN_SINGLE_QUOTES_REGEX),
+        text: txt
+      ).replace
+    end
+
+    def sub_punctuation_between_double_quotes(txt)
+      btwn_dbl_quote = txt.scan(BETWEEN_DOUBLE_QUOTES_REGEX)
+      PragmaticSegmenter::PunctuationReplacer.new(
+        matches_array: btwn_dbl_quote,
+        text: txt
+      ).replace
+    end
+
+    def sub_punctuation_between_quotes_arrow(txt)
+      PragmaticSegmenter::PunctuationReplacer.new(
+        matches_array: txt.scan(BETWEEN_QUOTE_ARROW_REGEX),
+        text: txt
+      ).replace
+    end
+
+    def sub_punctuation_between_quotes_slanted(txt)
+      PragmaticSegmenter::PunctuationReplacer.new(
+        matches_array: txt.scan(BETWEEN_QUOTE_SLANTED_REGEX),
+        text: txt
+      ).replace
+    end
+  end
+end

data/lib/pragmatic_segmenter/cleaner.rb

@@ -0,0 +1,141 @@
+# -*- encoding : utf-8 -*-
+
+module PragmaticSegmenter
+  module Rules
+    module HtmlRules
+      # Rubular: http://rubular.com/r/ENrVFMdJ8v
+      HTMLTagRule = Rule.new(/<\/?[^>]*>/, '')
+
+      # Rubular: http://rubular.com/r/XZVqMPJhea
+      EscapedHTMLTagRule = Rule.new(/&lt;\/?[^gt;]*gt;/, '')
+
+      All = [HTMLTagRule, EscapedHTMLTagRule]
+    end
+  end
+
+  # This is an opinionated class that removes errant newlines,
+  # xhtml, inline formatting, etc.
+  class Cleaner
+    include Rules
+    # Rubular: http://rubular.com/r/V57WnM9Zut
+    NewLineInMiddleOfWordRule = Rule.new(/\n(?=[a-zA-Z]{1,2}\n)/, '')
+
+    # Rubular: http://rubular.com/r/3GiRiP2IbD
+    NEWLINE_IN_MIDDLE_OF_SENTENCE_REGEX = /(?<=\s)\n(?=([a-z]|\())/
+
+    # Rubular: http://rubular.com/r/UZAVcwqck8
+    PDF_NewLineInMiddleOfSentenceRule = Rule.new(/(?<=[^\n]\s)\n(?=\S)/, '')
+
+    # Rubular: http://rubular.com/r/eaNwGavmdo
+    PDF_NewLineInMiddleOfSentenceNoSpacesRule = Rule.new(/\n(?=[a-z])/, ' ')
+
+    # Rubular: http://rubular.com/r/bAJrhyLNeZ
+    InlineFormattingRule = Rule.new(/\{b\^&gt;\d*&lt;b\^\}|\{b\^>\d*<b\^\}/, '')
+
+    # Rubular: http://rubular.com/r/dMxp5MixFS
+    DoubleNewLineWithSpaceRule = Rule.new(/\n \n/, "\r")
+
+    # Rubular: http://rubular.com/r/H6HOJeA8bq
+    DoubleNewLineRule = Rule.new(/\n\n/, "\r")
+
+    # Rubular: http://rubular.com/r/Gn18aAnLdZ
+    NewLineFollowedByBulletRule = Rule.new(/\n(?=•)/, "\r")
+
+    # Rubular: http://rubular.com/r/FseyMiiYFT
+    NewLineFollowedByPeriodRule = Rule.new(/\n(?=\.(\s|\n))/, '')
+
+    # Rubular: http://rubular.com/r/8mc1ArOIGy
+    TableOfContentsRule = Rule.new(/\.{5,}\s*\d+-*\d*/, "\r")
+
+    # Rubular: http://rubular.com/r/DwNSuZrNtk
+    ConsecutivePeriodsRule = Rule.new(/\.{5,}/, ' ')
+
+    ReplaceNewlineWithCarriageReturnRule = Rule.new(/\n/, "\r")
+
+    QuotationsFirstRule = Rule.new(/''/, '"')
+    QuotationsSecondRule = Rule.new(/``/, '"')
+
+    attr_reader :text, :doc_type
+    def initialize(text:, **args)
+      @text = Text.new(text.dup)
+      @doc_type = args[:doc_type]
+    end
+
+    # Clean text of unwanted formatting
+    #
+    # Example:
+    #   >> text = "This is a sentence\ncut off in the middle because pdf."
+    #   >> PragmaticSegmenter::Cleaner(text: text).clean
+    #   => "This is a sentence cut off in the middle because pdf."
+    #
+    # Arguments:
+    #    text:       (String)  *required
+    #    language:   (String)  *optional
+    #                (two-digit ISO 639-1 code e.g. 'en')
+    #    doc_type:   (String)  *optional
+    #                (e.g. 'pdf')
+
+    def clean
+      return unless text
+      @clean_text = remove_all_newlines(text)
+      @clean_text = replace_double_newlines(@clean_text)
+      @clean_text = replace_newlines(@clean_text)
+      @clean_text = @clean_text.apply(HtmlRules::All)
+      @clean_text = @clean_text.apply(InlineFormattingRule)
+      @clean_text = clean_quotations(@clean_text)
+      @clean_text = clean_table_of_contents(@clean_text)
+    end
+
+    private
+
+    def remove_all_newlines(txt)
+      clean_text = remove_newline_in_middle_of_sentence(txt)
+      remove_newline_in_middle_of_word(clean_text)
+    end
+
+    def remove_newline_in_middle_of_sentence(txt)
+      txt.dup.gsub!(/(?:[^\.])*/) do |match|
+        next unless match.include?("\n")
+        orig = match.dup
+        match.gsub!(NEWLINE_IN_MIDDLE_OF_SENTENCE_REGEX, '')
+        txt.gsub!(/#{Regexp.escape(orig)}/, "#{match}")
+      end
+      txt
+    end
+
+    def remove_newline_in_middle_of_word(txt)
+      txt.apply(NewLineInMiddleOfWordRule)
+    end
+
+    def replace_double_newlines(txt)
+      txt.apply(DoubleNewLineWithSpaceRule).
+          apply(DoubleNewLineRule)
+    end
+
+    def replace_newlines(txt)
+      if doc_type.eql?('pdf')
+        txt = remove_pdf_line_breaks(txt)
+      else
+        txt =
+          txt.apply(NewLineFollowedByPeriodRule).
+              apply(ReplaceNewlineWithCarriageReturnRule)
+      end
+      txt
+    end
+
+    def remove_pdf_line_breaks(txt)
+      txt.apply(NewLineFollowedByBulletRule).
+          apply(PDF_NewLineInMiddleOfSentenceRule).
+          apply(PDF_NewLineInMiddleOfSentenceNoSpacesRule)
+    end
+
+    def clean_quotations(txt)
+      txt.apply(QuotationsFirstRule).
+          apply(QuotationsSecondRule)
+    end
+
+    def clean_table_of_contents(txt)
+      txt.apply(TableOfContentsRule).apply(ConsecutivePeriodsRule)
+    end
+  end
+end

data/lib/pragmatic_segmenter/ellipsis.rb

@@ -0,0 +1,36 @@
+# -*- encoding : utf-8 -*-
+
+module PragmaticSegmenter
+  module Rules
+    # This class searches for ellipses within a string and
+    # replaces the periods.
+
+    # http://www.dailywritingtips.com/in-search-of-a-4-dot-ellipsis/
+    # http://www.thepunctuationguide.com/ellipses.html
+
+    module EllipsisRules
+      # Rubular: http://rubular.com/r/i60hCK81fz
+      ThreeConsecutiveRule = Rule.new(/\.\.\.(?=\s+[A-Z])/, '☏.')
+
+      # Rubular: http://rubular.com/r/Hdqpd90owl
+      FourConsecutiveRule = Rule.new(/(?<=\S)\.{3}(?=\.\s[A-Z])/, 'ƪ')
+
+      # Rubular: http://rubular.com/r/YBG1dIHTRu
+      ThreeSpaceRule = Rule.new(/(\s\.){3}\s/, '♟')
+
+      # Rubular: http://rubular.com/r/2VvZ8wRbd8
+      FourSpaceRule = Rule.new(/(?<=[a-z])(\.\s){3}\.(\z|$|\n)/, '♝')
+
+      OtherThreePeriodRule = Rule.new(/\.\.\./, 'ƪ')
+
+      All = [
+        ThreeSpaceRule,
+        FourSpaceRule,
+        FourConsecutiveRule,
+        ThreeConsecutiveRule,
+        OtherThreePeriodRule
+      ]
+
+    end
+  end
+end

data/lib/pragmatic_segmenter/exclamation_words.rb

@@ -0,0 +1,19 @@
+# -*- encoding : utf-8 -*-
+require 'pragmatic_segmenter/punctuation_replacer'
+
+module PragmaticSegmenter
+  # This class searches for exclamation points that
+  # are part of words and not ending punctuation and replaces them.
+  module ExclamationWords
+    WORDS_WITH_EXCLAMATIONS = ['!Xũ', '!Kung', 'ǃʼOǃKung', '!Xuun', '!Kung-Ekoka', 'ǃHu', 'ǃKhung', 'ǃKu', 'ǃung', 'ǃXo', 'ǃXû', 'ǃXung', 'ǃXũ', '!Xun', 'Yahoo!', 'Y!J', 'Yum!']
+
+    def self.apply_rules(text)
+      WORDS_WITH_EXCLAMATIONS.each do |exclamation|
+        PragmaticSegmenter::PunctuationReplacer.new(
+          matches_array: text.scan(/#{Regexp.escape(exclamation)}/),
+          text: text
+        ).replace
+      end
+    end
+  end
+end