pragmatic_segmenter 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9e26400933e02aca93c63db3221fb897f49cdf78
-  data.tar.gz: a0199ca43d0f8558ba3fe867c6e32c69787210c9
+  metadata.gz: a9cb6133aca84f8c6ff233ec6fb34b276cf47964
+  data.tar.gz: 00c1f664707e86e5c2ae5740c53acde5c814ece8
 SHA512:
-  metadata.gz: 1dca5e20b2b062070b0cd319da6549594add065b670506b7e7dd9cbfe9eb8c83985ef197bf6427ee496d700edab5a526c8165fc1a27567e553f7a5e625edb6fe
-  data.tar.gz: a53a4059b4ba41e7d40bf2a21ca1981ae5ed134a8017771f8ecd337bdc96762e2e78ec7c53c00c3ff2e3ac6179cf1e29f24fee703bc6b19fbbf31bc7ff572894
+  metadata.gz: d5726605fa78ec4067c79ed592a7983f2638b26a81fb88cf23bdffeb26d842c0eaed39a531181ecef6456218208f6d297e4316b36fc7f4a15f4deb2ebb7cb800
+  data.tar.gz: a1c99c7f3c73c1624a2b1d4792c8937dd27d3dfd4667dcb05844376d5ecad13ebe065f18fe3f40a61a4aa23baa3e8ab9b4dc3bc547821c6d9fc5700cd5a16f20

data/.travis.yml

@@ -2,6 +2,7 @@ language: ruby
 rvm:
   - "2.1.5"
   - "2.2.0"
+  - "2.2.4"
 # uncomment this line if your project needs to run something other than `rake`:
 # script: bundle exec rspec spec
 addons:

data/lib/pragmatic_segmenter/abbreviation_replacer.rb

@@ -8,20 +8,20 @@ module PragmaticSegmenter
     SENTENCE_STARTERS = %w(A Being Did For He How However I In It Millions More She That The There They We What When Where Who Why)
 
     attr_reader :text
-    def initialize(text:, language: Languages::Common)
+    def initialize(text:, language: )
       @text = Text.new(text)
       @language = language
     end
 
     def replace
-      @reformatted_text = text.apply(@language::PossessiveAbbreviationRule,
+      @text.apply(@language::PossessiveAbbreviationRule,
         @language::KommanditgesellschaftRule,
         @language::SingleLetterAbbreviationRules::All)
 
-      @reformatted_text = search_for_abbreviations_in_string(@reformatted_text)
-      @reformatted_text = replace_multi_period_abbreviations(@reformatted_text)
-      @reformatted_text = @reformatted_text.apply(@language::AmPmRules::All)
-      replace_abbreviation_as_sentence_boundary(@reformatted_text)
+      @text = search_for_abbreviations_in_string(@text)
+      @text = replace_multi_period_abbreviations(@text)
+      @text.apply(@language::AmPmRules::All)
+      replace_abbreviation_as_sentence_boundary(@text)
     end
 
     private

data/lib/pragmatic_segmenter/between_punctuation.rb

@@ -1,5 +1,4 @@
 # -*- encoding : utf-8 -*-
-require 'pragmatic_segmenter/punctuation_replacer'
 
 module PragmaticSegmenter
   # This class searches for punctuation between quotes or parenthesis
@@ -66,13 +65,16 @@ module PragmaticSegmenter
     end
 
     def sub_punctuation_between_double_quotes(txt)
-      btwn_dbl_quote = txt.scan(BETWEEN_DOUBLE_QUOTES_REGEX)
       PragmaticSegmenter::PunctuationReplacer.new(
-        matches_array: btwn_dbl_quote,
+        matches_array: btwn_dbl_quote(txt),
         text: txt
       ).replace
     end
 
+    def btwn_dbl_quote(txt)
+      txt.scan(BETWEEN_DOUBLE_QUOTES_REGEX)
+    end
+
     def sub_punctuation_between_quotes_arrow(txt)
       PragmaticSegmenter::PunctuationReplacer.new(
         matches_array: txt.scan(BETWEEN_QUOTE_ARROW_REGEX),
@@ -87,4 +89,4 @@ module PragmaticSegmenter
       ).replace
     end
   end
-end
+end

data/lib/pragmatic_segmenter/cleaner.rb

@@ -1,4 +1,5 @@
 # -*- encoding : utf-8 -*-
+require_relative 'cleaner/rules'
 
 module PragmaticSegmenter
   # This is an opinionated class that removes errant newlines,
@@ -7,8 +8,8 @@ module PragmaticSegmenter
     include Rules
 
     attr_reader :text, :doc_type
-    def initialize(text:, doc_type: nil, language: Languages::Common, **args)
-      @text = Text.new(text.dup)
+    def initialize(text:, doc_type: nil, language: Languages::Common)
+      @text = Text.new(text)
       @doc_type = doc_type
       @language = language
     end
@@ -29,17 +30,19 @@ module PragmaticSegmenter
 
     def clean
       return unless text
-      @clean_text = remove_all_newlines(text)
-      replace_double_newlines(@clean_text)
-      replace_newlines(@clean_text)
-      replace_escaped_newlines(@clean_text)
-      @clean_text.apply(HTMLRules::All)
-      replace_punctuation_in_brackets(@clean_text)
-      @clean_text.apply(InlineFormattingRule)
-      clean_quotations(@clean_text)
-      clean_table_of_contents(@clean_text)
-      check_for_no_space_in_between_sentences(@clean_text)
-      clean_consecutive_characters(@clean_text)
+      remove_all_newlines
+      replace_double_newlines
+      replace_newlines
+      replace_escaped_newlines
+
+      @text.apply(HTML::All)
+
+      replace_punctuation_in_brackets
+      @text.apply(InlineFormattingRule)
+      clean_quotations
+      clean_table_of_contents
+      check_for_no_space_in_between_sentences
+      clean_consecutive_characters
     end
 
     private
@@ -48,18 +51,18 @@ module PragmaticSegmenter
       @language::Abbreviation::ABBREVIATIONS
     end
 
-    def check_for_no_space_in_between_sentences(txt)
-      words = txt.split(' ')
+    def check_for_no_space_in_between_sentences
+      words = @text.split(' ')
       words.each do |word|
-        search_for_connected_sentences(word, txt, NO_SPACE_BETWEEN_SENTENCES_REGEX, NoSpaceBetweenSentencesRule)
-        search_for_connected_sentences(word, txt, NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX, NoSpaceBetweenSentencesDigitRule)
+        search_for_connected_sentences(word, @text, NO_SPACE_BETWEEN_SENTENCES_REGEX, NoSpaceBetweenSentencesRule)
+        search_for_connected_sentences(word, @text, NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX, NoSpaceBetweenSentencesDigitRule)
       end
-      txt
+      @text
     end
 
-    def replace_punctuation_in_brackets(txt)
-      txt.dup.gsub!(/\[(?:[^\]])*\]/) do |match|
-        txt.gsub!(/#{Regexp.escape(match)}/, "#{match.dup.gsub!(/\?/, '&ᓷ&')}") if match.include?('?')
+    def replace_punctuation_in_brackets
+      @text.dup.gsub!(/\[(?:[^\]])*\]/) do |match|
+        @text.gsub!(/#{Regexp.escape(match)}/, "#{match.dup.gsub!(/\?/, '&ᓷ&')}") if match.include?('?')
       end
     end
 
@@ -74,60 +77,61 @@ module PragmaticSegmenter
       end
     end
 
-    def remove_all_newlines(txt)
-      clean_text = remove_newline_in_middle_of_sentence(txt)
-      remove_newline_in_middle_of_word(clean_text)
+    def remove_all_newlines
+      remove_newline_in_middle_of_sentence
+      remove_newline_in_middle_of_word
     end
 
-    def remove_newline_in_middle_of_sentence(txt)
-      txt.dup.gsub!(/(?:[^\.])*/) do |match|
+    def remove_newline_in_middle_of_sentence
+      @text.dup.gsub!(/(?:[^\.])*/) do |match|
         next unless match.include?("\n")
         orig = match.dup
         match.gsub!(NEWLINE_IN_MIDDLE_OF_SENTENCE_REGEX, '')
-        txt.gsub!(/#{Regexp.escape(orig)}/, "#{match}")
+        @text.gsub!(/#{Regexp.escape(orig)}/, "#{match}")
       end
-      txt
+      @text
     end
 
-    def remove_newline_in_middle_of_word(txt)
-      txt.apply NewLineInMiddleOfWordRule
+    def remove_newline_in_middle_of_word
+      @text.apply NewLineInMiddleOfWordRule
     end
 
-    def replace_escaped_newlines(txt)
-      txt.apply EscapedNewLineRule, EscapedCarriageReturnRule,
+    def replace_escaped_newlines
+      @text.apply EscapedNewLineRule, EscapedCarriageReturnRule,
         TypoEscapedNewLineRule, TypoEscapedCarriageReturnRule
     end
 
-    def replace_double_newlines(txt)
-      txt.apply DoubleNewLineWithSpaceRule, DoubleNewLineRule
+    def replace_double_newlines
+      @text.apply DoubleNewLineWithSpaceRule, DoubleNewLineRule
     end
 
-    def replace_newlines(txt)
+    def replace_newlines
       if doc_type.eql?('pdf')
-        remove_pdf_line_breaks(txt)
+        remove_pdf_line_breaks
       else
-        txt.apply NewLineFollowedByPeriodRule,
+        @text.apply NewLineFollowedByPeriodRule,
           ReplaceNewlineWithCarriageReturnRule
       end
     end
 
-    def remove_pdf_line_breaks(txt)
-      txt.apply NewLineFollowedByBulletRule,
-        PDF_NewLineInMiddleOfSentenceRule,
-        PDF_NewLineInMiddleOfSentenceNoSpacesRule
+    def remove_pdf_line_breaks
+      @text.apply NewLineFollowedByBulletRule,
+
+        PDF::NewLineInMiddleOfSentenceRule,
+        PDF::NewLineInMiddleOfSentenceNoSpacesRule
     end
 
-    def clean_quotations(txt)
-      txt.apply QuotationsFirstRule, QuotationsSecondRule
+    def clean_quotations
+      @text.apply QuotationsFirstRule, QuotationsSecondRule
     end
 
-    def clean_table_of_contents(txt)
-      txt.apply TableOfContentsRule, ConsecutivePeriodsRule,
+    def clean_table_of_contents
+      @text.apply TableOfContentsRule, ConsecutivePeriodsRule,
         ConsecutiveForwardSlashRule
     end
 
-    def clean_consecutive_characters(txt)
-      txt.apply ConsecutivePeriodsRule, ConsecutiveForwardSlashRule
+    def clean_consecutive_characters
+      @text.apply ConsecutivePeriodsRule, ConsecutiveForwardSlashRule
     end
   end
 end

data/lib/pragmatic_segmenter/cleaner/rules.rb

@@ -0,0 +1,86 @@
+module PragmaticSegmenter
+  # This is an opinionated class that removes errant newlines,
+  # xhtml, inline formatting, etc.
+  class Cleaner
+    module Rules
+      # Rubular: http://rubular.com/r/V57WnM9Zut
+      NewLineInMiddleOfWordRule = Rule.new(/\n(?=[a-zA-Z]{1,2}\n)/, '')
+
+      # Rubular: http://rubular.com/r/dMxp5MixFS
+      DoubleNewLineWithSpaceRule = Rule.new(/\n \n/, "\r")
+
+      # Rubular: http://rubular.com/r/H6HOJeA8bq
+      DoubleNewLineRule = Rule.new(/\n\n/, "\r")
+
+      # Rubular: http://rubular.com/r/FseyMiiYFT
+      NewLineFollowedByPeriodRule = Rule.new(/\n(?=\.(\s|\n))/, '')
+
+
+      ReplaceNewlineWithCarriageReturnRule = Rule.new(/\n/, "\r")
+
+      EscapedNewLineRule = Rule.new(/\\n/, "\n")
+      EscapedCarriageReturnRule = Rule.new(/\\r/, "\r")
+
+      TypoEscapedNewLineRule = Rule.new(/\\\ n/, "\n")
+
+      TypoEscapedCarriageReturnRule = Rule.new(/\\\ r/, "\r")
+
+
+
+
+      # Rubular: http://rubular.com/r/bAJrhyLNeZ
+      InlineFormattingRule = Rule.new(/\{b\^&gt;\d*&lt;b\^\}|\{b\^>\d*<b\^\}/, '')
+
+      # Rubular: http://rubular.com/r/8mc1ArOIGy
+      TableOfContentsRule = Rule.new(/\.{5,}\s*\d+-*\d*/, "\r")
+
+      # Rubular: http://rubular.com/r/DwNSuZrNtk
+      ConsecutivePeriodsRule = Rule.new(/\.{5,}/, ' ')
+
+      # Rubular: http://rubular.com/r/IQ4TPfsbd8
+      ConsecutiveForwardSlashRule = Rule.new(/\/{3}/, '')
+
+
+      # Rubular: http://rubular.com/r/6dt98uI76u
+      NO_SPACE_BETWEEN_SENTENCES_REGEX = /(?<=[a-z])\.(?=[A-Z])/
+      NoSpaceBetweenSentencesRule = Rule.new(NO_SPACE_BETWEEN_SENTENCES_REGEX, '. ')
+
+      # Rubular: http://rubular.com/r/l6KN6rH5XE
+      NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX = /(?<=\d)\.(?=[A-Z])/
+      NoSpaceBetweenSentencesDigitRule = Rule.new(NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX, '. ')
+
+
+      URL_EMAIL_KEYWORDS = ['@', 'http', '.com', 'net', 'www', '//']
+
+      # Rubular: http://rubular.com/r/3GiRiP2IbD
+      NEWLINE_IN_MIDDLE_OF_SENTENCE_REGEX = /(?<=\s)\n(?=([a-z]|\())/
+
+
+      # Rubular: http://rubular.com/r/Gn18aAnLdZ
+      NewLineFollowedByBulletRule = Rule.new(/\n(?=•)/, "\r")
+
+      QuotationsFirstRule = Rule.new(/''/, '"')
+      QuotationsSecondRule = Rule.new(/``/, '"')
+
+
+      module HTML
+        # Rubular: http://rubular.com/r/ENrVFMdJ8v
+        HTMLTagRule = Rule.new(/<\/?[^>]*>/, '')
+
+        # Rubular: http://rubular.com/r/XZVqMPJhea
+        EscapedHTMLTagRule = Rule.new(/&lt;\/?[^gt;]*gt;/, '')
+
+        All = [HTMLTagRule, EscapedHTMLTagRule]
+      end
+
+      module PDF
+        # Rubular: http://rubular.com/r/UZAVcwqck8
+        NewLineInMiddleOfSentenceRule = Rule.new(/(?<=[^\n]\s)\n(?=\S)/, '')
+
+        # Rubular: http://rubular.com/r/eaNwGavmdo
+        NewLineInMiddleOfSentenceNoSpacesRule = Rule.new(/\n(?=[a-z])/, ' ')
+      end
+
+    end
+  end
+end

data/lib/pragmatic_segmenter/languages.rb

@@ -1,7 +1,6 @@
 require 'pragmatic_segmenter/types'
-require 'pragmatic_segmenter/process'
+require 'pragmatic_segmenter/processor'
 require 'pragmatic_segmenter/cleaner'
-require 'pragmatic_segmenter/rules'
 
 require 'pragmatic_segmenter/languages/common'
 
@@ -27,36 +26,28 @@ require 'pragmatic_segmenter/languages/chinese'
 module PragmaticSegmenter
   module Languages
     LANGUAGE_CODES = {
-      'en' => 'English',
-      'de' => 'Deutsch',
-      'es' => 'Spanish',
-      'fr' => 'French',
-      'it' => 'Italian',
-      'ja' => 'Japanese',
-      'el' => 'Greek',
-      'ru' => 'Russian',
-      'ar' => 'Arabic',
-      'am' => 'Amharic',
-      'hi' => 'Hindi',
-      'hy' => 'Armenian',
-      'fa' => 'Persian',
-      'my' => 'Burmese',
-      'ur' => 'Urdu',
-      'nl' => 'Dutch',
-      'pl' => 'Polish',
-      'zh' => 'Chinese',
+      'en' => English,
+      'de' => Deutsch,
+      'es' => Spanish,
+      'fr' => French,
+      'it' => Italian,
+      'ja' => Japanese,
+      'el' => Greek,
+      'ru' => Russian,
+      'ar' => Arabic,
+      'am' => Amharic,
+      'hi' => Hindi,
+      'hy' => Armenian,
+      'fa' => Persian,
+      'my' => Burmese,
+      'ur' => Urdu,
+      'nl' => Dutch,
+      'pl' => Polish,
+      'zh' => Chinese,
     }
 
-    def process_class
-      language_module::Process
-    end
-
-    def cleaner_class
-      language_module::Cleaner
-    end
-
-    def language_module
-      Object.const_get("PragmaticSegmenter::Languages::#{LANGUAGE_CODES[language] || 'Common'}")
+    def self.get_language_by_code(code)
+      LANGUAGE_CODES[code] || Common
     end
   end
 end

data/lib/pragmatic_segmenter/languages/arabic.rb

@@ -18,19 +18,6 @@ module PragmaticSegmenter
       # Rubular: http://rubular.com/r/kPRgApNHUg
       ReplaceNonSentenceBoundaryCommaRule = Rule.new(/،(?=\s\S+،)/, '♬')
 
-      class Process < Process
-        private
-
-        def sentence_boundary_punctuation(txt)
-          txt = txt.apply(ReplaceColonBetweenNumbersRule, ReplaceNonSentenceBoundaryCommaRule)
-          txt.scan(SENTENCE_BOUNDARY_REGEX)
-        end
-
-        def replace_abbreviations(txt)
-          AbbreviationReplacer.new(text: txt, language: Arabic).replace
-        end
-      end
-
       class AbbreviationReplacer  < AbbreviationReplacer
         private
 

data/lib/pragmatic_segmenter/languages/common.rb

@@ -1,3 +1,6 @@
+require_relative 'common/numbers'
+require_relative 'common/ellipsis'
+
 module PragmaticSegmenter
   module Languages
     module Common
@@ -11,69 +14,89 @@ module PragmaticSegmenter
         NUMBER_ABBREVIATIONS = ['art', 'ext', 'no', 'nos', 'p', 'pp']
       end
 
-      SENTENCE_BOUNDARY_REGEX = /\u{ff08}(?:[^\u{ff09}])*\u{ff09}(?=\s?[A-Z])|\u{300c}(?:[^\u{300d}])*\u{300d}(?=\s[A-Z])|\((?:[^\)]){2,}\)(?=\s[A-Z])|'(?:[^'])*[^,]'(?=\s[A-Z])|"(?:[^"])*[^,]"(?=\s[A-Z])|“(?:[^”])*[^,]”(?=\s[A-Z])|\S.*?[。．.！!?？ȸȹ☉☈☇☄]/
-
-      include Rules
-      # Rubular: http://rubular.com/r/NqCqv372Ix
-      QUOTATION_AT_END_OF_SENTENCE_REGEX = /[!?\.-][\"\'\u{201d}\u{201c}]\s{1}[A-Z]/
+      module Abbreviations
+        # Rubular: http://rubular.com/r/EUbZCNfgei
+        WithMultiplePeriodsAndEmailRule = Rule.new(/(\w)(\.)(\w)/, '\1∮\3')
+      end
 
-      # Rubular: http://rubular.com/r/6flGnUMEVl
-      PARENS_BETWEEN_DOUBLE_QUOTES_REGEX = /["”]\s\(.*\)\s["“]/
+      # Rubular: http://rubular.com/r/G2opjedIm9
+      GeoLocationRule = Rule.new(/(?<=[a-zA-z]°)\.(?=\s*\d+)/, '∯')
 
-      # Rubular: http://rubular.com/r/TYzr4qOW1Q
-      BETWEEN_DOUBLE_QUOTES_REGEX = /"(?:[^"])*[^,]"|“(?:[^”])*[^,]”/
+      SingleNewLineRule = Rule.new(/\n/, 'ȹ')
 
-      # Rubular: http://rubular.com/r/JMjlZHAT4g
-      SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX = /(?<=[!?\.-][\"\'\u{201d}\u{201c}])\s{1}(?=[A-Z])/
+      module DoublePunctuationRules
+        FirstRule = Rule.new(/\?!/, '☉')
+        SecondRule = Rule.new(/!\?/, '☈')
+        ThirdRule = Rule.new(/\?\?/, '☇')
+        ForthRule = Rule.new(/!!/, '☄')
 
-      # Rubular: http://rubular.com/r/mQ8Es9bxtk
-      CONTINUOUS_PUNCTUATION_REGEX = /(?<=\S)(!|\?){3,}(?=(\s|\z|$))/
+        All = [ FirstRule, SecondRule, ThirdRule, ForthRule ]
+      end
 
-      # Rubular: http://rubular.com/r/yqa4Rit8EY
-      PossessiveAbbreviationRule = Rule.new(/\.(?='s\s)|\.(?='s$)|\.(?='s\z)/, '∯')
 
-      # Rubular: http://rubular.com/r/NEv265G2X2
-      KommanditgesellschaftRule = Rule.new(/(?<=Co)\.(?=\sKG)/, '∯')
+      # Rubular: http://rubular.com/r/aXPUGm6fQh
+      QuestionMarkInQuotationRule = Rule.new(/\?(?=(\'|\"))/, '&ᓷ&')
 
-      # Rubular: http://rubular.com/r/xDkpFZ0EgH
-      MULTI_PERIOD_ABBREVIATION_REGEX = /\b[a-z](?:\.[a-z])+[.]/i
 
-      module AmPmRules
-        # Rubular: http://rubular.com/r/Vnx3m4Spc8
-        UpperCasePmRule = Rule.new(/(?<=P∯M)∯(?=\s[A-Z])/, '.')
+      module ExclamationPointRules
+        # Rubular: http://rubular.com/r/XS1XXFRfM2
+        InQuotationRule = Rule.new(/\!(?=(\'|\"))/, '&ᓴ&')
 
-        # Rubular: http://rubular.com/r/AJMCotJVbW
-        UpperCaseAmRule = Rule.new(/(?<=A∯M)∯(?=\s[A-Z])/, '.')
+        # Rubular: http://rubular.com/r/sl57YI8LkA
+        BeforeCommaMidSentenceRule = Rule.new(/\!(?=\,\s[a-z])/, '&ᓴ&')
 
-        # Rubular: http://rubular.com/r/13q7SnOhgA
-        LowerCasePmRule = Rule.new(/(?<=p∯m)∯(?=\s[A-Z])/, '.')
+        # Rubular: http://rubular.com/r/f9zTjmkIPb
+        MidSentenceRule = Rule.new(/\!(?=\s[a-z])/, '&ᓴ&')
 
-        # Rubular: http://rubular.com/r/DgUDq4mLz5
-        LowerCaseAmRule = Rule.new(/(?<=a∯m)∯(?=\s[A-Z])/, '.')
+        All = [ InQuotationRule, BeforeCommaMidSentenceRule, MidSentenceRule ]
+      end
 
-        All = [UpperCasePmRule, UpperCaseAmRule, LowerCasePmRule, LowerCaseAmRule]
+      module SubSymbolsRules
+        Period = Rule.new(/∯/, '.')
+        ArabicComma = Rule.new(/♬/, '،')
+        SemiColon = Rule.new(/♭/, ':')
+        FullWidthPeriod = Rule.new(/&ᓰ&/, '。')
+        SpecialPeriod = Rule.new(/&ᓱ&/, '．')
+        FullWidthExclamation = Rule.new(/&ᓳ&/, '！')
+        ExclamationPoint = Rule.new(/&ᓴ&/, '!')
+        QuestionMark = Rule.new(/&ᓷ&/, '?')
+        FullWidthQuestionMark = Rule.new(/&ᓸ&/, '？')
+        MixedDoubleQE = Rule.new(/☉/, '?!')
+        MixedDoubleQQ = Rule.new(/☇/, '??')
+        MixedDoubleEQ = Rule.new(/☈/, '!?')
+        MixedDoubleEE = Rule.new(/☄/, '!!')
+        LeftParens = Rule.new(/&✂&/, '(')
+        RightParens = Rule.new(/&⌬&/, ')')
+        TemporaryEndingPunctutation = Rule.new('ȸ', '')
+        Newline = Rule.new(/ȹ/, "\n")
+
+        All = [ Period, ArabicComma,
+                SemiColon, FullWidthPeriod,
+                SpecialPeriod, FullWidthExclamation,
+                ExclamationPoint, QuestionMark,
+                FullWidthQuestionMark, MixedDoubleQE,
+                MixedDoubleQQ, MixedDoubleEQ,
+                MixedDoubleEE, LeftParens,
+                RightParens, TemporaryEndingPunctutation,
+                Newline ]
       end
 
-      # This class searches for periods within an abbreviation and
-      # replaces the periods.
-      module SingleLetterAbbreviationRules
-        # Rubular: http://rubular.com/r/e3H6kwnr6H
-        SingleUpperCaseLetterAtStartOfLineRule = Rule.new(/(?<=^[A-Z])\.(?=\s)/, '∯')
 
-        # Rubular: http://rubular.com/r/gitvf0YWH4
-        SingleUpperCaseLetterRule = Rule.new(/(?<=\s[A-Z])\.(?=\s)/, '∯')
+      module ReinsertEllipsisRules
+        SubThreeConsecutivePeriod = Rule.new(/ƪ/, '...')
+        SubThreeSpacePeriod = Rule.new(/♟/, ' . . . ')
+        SubFourSpacePeriod = Rule.new(/♝/, '. . . .')
+        SubTwoConsecutivePeriod = Rule.new(/☏/, '..')
+        SubOnePeriod = Rule.new(/∮/, '.')
 
-        All = [
-          SingleUpperCaseLetterAtStartOfLineRule,
-          SingleUpperCaseLetterRule
-        ]
+        All = [ SubThreeConsecutivePeriod, SubThreeSpacePeriod,
+                SubFourSpacePeriod, SubTwoConsecutivePeriod,
+                SubOnePeriod ]
       end
 
+      ExtraWhiteSpaceRule = Rule.new(/\s{3,}/, ' ')
 
-      class Process < PragmaticSegmenter::Process
-      end
-      class Cleaner < PragmaticSegmenter::Cleaner
-      end
+      SubSingleQuoteRule = Rule.new(/&⎋&/, "'")
     end
   end
 end