pragmatic_segmenter 0.0.1

data/lib/pragmatic_segmenter/number.rb

@@ -0,0 +1,35 @@
+# -*- encoding : utf-8 -*-
+
+module PragmaticSegmenter
+  # This class searches for numbers with periods within a string and
+  # replaces the periods.
+  class Number
+    # Rubular: http://rubular.com/r/oNyxBOqbyy
+    PeriodBeforeNumberRule = Rule.new(/\.(?=\d)/, '∯')
+
+    # Rubular: http://rubular.com/r/EMk5MpiUzt
+    NumberAfterPeriodBeforeLetterRule = Rule.new(/(?<=\d)\.(?=\S)/, '∯')
+
+    # Rubular: http://rubular.com/r/rf4l1HjtjG
+    NewLineNumberPeriodSpaceLetterRule = Rule.new(/(?<=\r\d)\.(?=(\s\S)|\))/, '∯')
+
+    # Rubular: http://rubular.com/r/HPa4sdc6b9
+    StartLineNumberPeriodRule = Rule.new(/(?<=^\d)\.(?=(\s\S)|\))/, '∯')
+
+    # Rubular: http://rubular.com/r/NuvWnKleFl
+    StartLineTwoDigitNumberPeriodRule = Rule.new(/(?<=^\d\d)\.(?=(\s\S)|\))/, '∯')
+
+    attr_reader :text
+    def initialize(text:)
+      @text = Text.new(text)
+    end
+
+    def replace
+      @formatted_text = @text.apply(PeriodBeforeNumberRule).
+          apply(NumberAfterPeriodBeforeLetterRule).
+          apply(NewLineNumberPeriodSpaceLetterRule).
+          apply(StartLineNumberPeriodRule).
+          apply(StartLineTwoDigitNumberPeriodRule)
+    end
+  end
+end

data/lib/pragmatic_segmenter/process.rb

@@ -0,0 +1,126 @@
+# -*- encoding : utf-8 -*-
+require 'pragmatic_segmenter/list'
+require 'pragmatic_segmenter/abbreviation_replacer'
+require 'pragmatic_segmenter/number'
+require 'pragmatic_segmenter/ellipsis'
+require 'pragmatic_segmenter/exclamation_words'
+require 'pragmatic_segmenter/punctuation_replacer'
+require 'pragmatic_segmenter/between_punctuation'
+require 'pragmatic_segmenter/sentence_boundary_punctuation'
+require 'pragmatic_segmenter/punctuation'
+
+module PragmaticSegmenter
+  # This class processing segmenting the text.
+  class Process
+    include Rules
+    # Rubular: http://rubular.com/r/NqCqv372Ix
+    QUOTATION_AT_END_OF_SENTENCE_REGEX = /[!?\.][\"\'\u{201d}\u{201c}]\s{1}[A-Z]/
+
+    # Rubular: http://rubular.com/r/JMjlZHAT4g
+    SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX = /(?<=[!?\.][\"\'\u{201d}\u{201c}])\s{1}(?=[A-Z])/
+
+    attr_reader :text, :doc_type
+    def initialize(text:, doc_type:)
+      @text = text
+      @doc_type = doc_type
+    end
+
+    def process
+      reformatted_text = PragmaticSegmenter::List.new(text: text).add_line_break
+      reformatted_text = replace_abbreviations(reformatted_text)
+      reformatted_text = replace_numbers(reformatted_text)
+      reformatted_text = reformatted_text.apply(GeoLocationRule)
+      split_lines(reformatted_text)
+    end
+
+    private
+
+    def split_lines(txt)
+      segments = []
+      lines = txt.split("\r")
+      lines.each do |l|
+        next if l.eql?('')
+        analyze_lines(l, segments, punctuation_array)
+      end
+      sentence_array = []
+      segments.each_with_index do |line|
+        next if line.gsub(/_{3,}/, '').length.eql?(0) || line.length < 2
+        line = reinsert_ellipsis(line)
+        line = line.apply(ExtraWhiteSpaceRule)
+        if line =~ QUOTATION_AT_END_OF_SENTENCE_REGEX
+          subline = line.split(SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX)
+          subline.each do |s|
+            sentence_array << s
+          end
+        else
+          sentence_array << line.tr("\n", '').strip
+        end
+      end
+      sentence_array.reject(&:empty?)
+    end
+
+    def analyze_lines(line, segments, punctuation)
+      line = line.apply(SingleNewLineRule, EllipsisRules::All, EmailRule)
+      clause_1 = false
+      end_punc_check = false
+      punctuation.each do |p|
+        end_punc_check = true if line[-1].include?(p)
+        clause_1 = true if line.include?(p)
+      end
+      if clause_1
+        segments = process_text(line, end_punc_check, segments)
+      else
+        line.gsub!(/ȹ/, "\n")
+        line.gsub!(/∯/, '.')
+        segments << line
+      end
+    end
+
+    def process_text(line, end_punc_check, segments)
+      line << 'ȸ' if !end_punc_check
+      PragmaticSegmenter::ExclamationWords.apply_rules(line)
+      between_punctutation(line)
+      line = line.apply(
+        DoublePuctationRules::All,
+        QuestionMarkInQuotationRule,
+        ExclamationPointRules::All
+      )
+      subline = sentence_boundary_punctuation(line)
+      subline.each_with_index do |s_l|
+        segments << sub_symbols(s_l)
+      end
+    end
+
+    def replace_numbers(txt)
+      PragmaticSegmenter::Number.new(text: txt).replace
+    end
+
+    def replace_abbreviations(txt)
+      PragmaticSegmenter::AbbreviationReplacer.new(text: txt).replace
+    end
+
+    def punctuation_array
+      PragmaticSegmenter::Punctuation.new.punct
+    end
+
+    def between_punctutation(txt)
+      PragmaticSegmenter::BetweenPunctuation.new(text: txt).replace
+    end
+
+    def sentence_boundary_punctuation(txt)
+      PragmaticSegmenter::SentenceBoundaryPunctuation.new(text: txt).split
+    end
+
+    def sub_symbols(txt)
+      txt.gsub(/∯/, '.').gsub(/♬/, '،').gsub(/♭/, ':').gsub(/ᓰ/, '。').gsub(/ᓱ/, '．')
+        .gsub(/ᓳ/, '！').gsub(/ᓴ/, '!').gsub(/ᓷ/, '?').gsub(/ᓸ/, '？').gsub(/☉/, '?!')
+        .gsub(/☈/, '!?').gsub(/☇/, '??').gsub(/☄/, '!!').delete('ȸ').gsub(/ȹ/, "\n")
+    end
+
+    def reinsert_ellipsis(line)
+      line.gsub(/ƪ/, '...').gsub(/♟/, ' . . . ')
+        .gsub(/♝/, '. . . .').gsub(/☏/, '..')
+        .gsub(/∮/, '.')
+    end
+  end
+end

data/lib/pragmatic_segmenter/punctuation.rb

@@ -0,0 +1,12 @@
+# -*- encoding : utf-8 -*-
+
+module PragmaticSegmenter
+  # This class holds the punctuation marks.
+  class Punctuation
+    PUNCT = ['。', '．', '.', '！', '!', '?', '？']
+
+    def punct
+      PUNCT
+    end
+  end
+end

data/lib/pragmatic_segmenter/punctuation_replacer.rb

@@ -0,0 +1,62 @@
+# -*- encoding : utf-8 -*-
+
+module PragmaticSegmenter
+  # This class replaces punctuation that is typically a sentence boundary
+  # but in this case is not a sentence boundary.
+  class PunctuationReplacer
+    attr_reader :matches_array, :text
+    def initialize(text:, matches_array:)
+      @text = text
+      @matches_array = matches_array
+    end
+
+    def replace
+      replace_punctuation(matches_array, text)
+    end
+
+    private
+
+    def replace_punctuation(array, txt)
+      return if !array || array.empty?
+      txt.gsub!('(', '\\(')
+      txt.gsub!(')', '\\)')
+      txt.gsub!(']', '\\]')
+      txt.gsub!('[', '\\[')
+      txt.gsub!('-', '\\-')
+      array.each do |a|
+        a.gsub!('(', '\\(')
+        a.gsub!(')', '\\)')
+        a.gsub!(']', '\\]')
+        a.gsub!('[', '\\[')
+        a.gsub!('-', '\\-')
+
+        sub = a.gsub('.', '∯')
+        txt.gsub!(/#{Regexp.escape(a)}/, "#{sub}")
+
+        sub_1 = sub.gsub('。', 'ᓰ')
+        txt.gsub!(/#{Regexp.escape(sub)}/, "#{sub_1}")
+
+        sub_2 = sub_1.gsub('．', 'ᓱ')
+        txt.gsub!(/#{Regexp.escape(sub_1)}/, "#{sub_2}")
+
+        sub_3 = sub_2.gsub('！', 'ᓳ')
+        txt.gsub!(/#{Regexp.escape(sub_2)}/, "#{sub_3}")
+
+        sub_4 = sub_3.gsub('!', 'ᓴ')
+        txt.gsub!(/#{Regexp.escape(sub_3)}/, "#{sub_4}")
+
+        sub_5 = sub_4.gsub('?', 'ᓷ')
+        txt.gsub!(/#{Regexp.escape(sub_4)}/, "#{sub_5}")
+
+        sub_6 = sub_5.gsub('？', 'ᓸ')
+        txt.gsub!(/#{Regexp.escape(sub_5)}/, "#{sub_6}")
+      end
+      txt.gsub!('\\(', '(')
+      txt.gsub!('\\)', ')')
+      txt.gsub!('\\[', '[')
+      txt.gsub!('\\]', ']')
+      txt.gsub!('\\-', '-')
+      txt
+    end
+  end
+end

data/lib/pragmatic_segmenter/rules.rb

@@ -0,0 +1,38 @@
+module PragmaticSegmenter
+  module Rules
+    # Rubular: http://rubular.com/r/EUbZCNfgei
+    EmailRule = Rule.new(/(\w)(\.)(\w)/, '\1∮\3')
+
+    # Rubular: http://rubular.com/r/G2opjedIm9
+    GeoLocationRule = Rule.new(/(?<=[a-zA-z]°)\.(?=\s*\d+)/, '∯')
+
+    SingleNewLineRule = Rule.new(/\n/, 'ȹ')
+
+    ExtraWhiteSpaceRule = Rule.new(/\s{3,}/, ' ')
+
+    # Rubular: http://rubular.com/r/aXPUGm6fQh
+    QuestionMarkInQuotationRule = Rule.new(/\?(?=(\'|\"))/, 'ᓷ')
+
+    module ExclamationPointRules
+      # Rubular: http://rubular.com/r/XS1XXFRfM2
+      InQuotationRule = Rule.new(/\!(?=(\'|\"))/, 'ᓴ')
+
+      # Rubular: http://rubular.com/r/sl57YI8LkA
+      BeforeCommaMidSentenceRule = Rule.new(/\!(?=\,\s[a-z])/, 'ᓴ')
+
+      # Rubular: http://rubular.com/r/f9zTjmkIPb
+      MidSentenceRule = Rule.new(/\!(?=\s[a-z])/, 'ᓴ')
+
+      All = [ InQuotationRule, BeforeCommaMidSentenceRule, MidSentenceRule ]
+    end
+
+    module DoublePuctationRules
+      FirstRule = Rule.new(/\?!/, '☉')
+      SecondRule = Rule.new(/!\?/, '☈')
+      ThirdRule = Rule.new(/\?\?/, '☇')
+      ForthRule = Rule.new(/!!/, '☄')
+
+      All = [ FirstRule, SecondRule, ThirdRule, ForthRule ]
+    end
+  end
+end

data/lib/pragmatic_segmenter/segmenter.rb

@@ -0,0 +1,81 @@
+# -*- encoding : utf-8 -*-
+require 'pragmatic_segmenter/types'
+require 'pragmatic_segmenter/process'
+require 'pragmatic_segmenter/cleaner'
+require 'pragmatic_segmenter/languages/english'
+require 'pragmatic_segmenter/languages/deutsch'
+require 'pragmatic_segmenter/languages/hindi'
+require 'pragmatic_segmenter/languages/persian'
+require 'pragmatic_segmenter/languages/amharic'
+require 'pragmatic_segmenter/languages/arabic'
+require 'pragmatic_segmenter/languages/greek'
+require 'pragmatic_segmenter/languages/armenian'
+require 'pragmatic_segmenter/languages/burmese'
+require 'pragmatic_segmenter/languages/urdu'
+require 'pragmatic_segmenter/languages/french'
+require 'pragmatic_segmenter/languages/italian'
+require 'pragmatic_segmenter/languages/spanish'
+require 'pragmatic_segmenter/languages/russian'
+require 'pragmatic_segmenter/languages/japanese'
+require 'pragmatic_segmenter/rules'
+
+module PragmaticSegmenter
+  # This class segments a text into an array of sentences.
+  class Segmenter
+    include Rules
+    attr_reader :text, :language, :doc_type
+    def initialize(text:, **args)
+      return [] unless text
+      @language = args[:language] || 'en'
+      @doc_type = args[:doc_type]
+      if args[:clean].eql?(false)
+        @text = text.dup
+      else
+        case @language
+        when 'en'
+          @text = PragmaticSegmenter::Languages::English::Cleaner.new(text: text.dup, doc_type: args[:doc_type]).clean
+        when 'ja'
+          @text = PragmaticSegmenter::Languages::Japanese::Cleaner.new(text: text.dup, doc_type: args[:doc_type]).clean
+        else
+          @text = PragmaticSegmenter::Cleaner.new(text: text.dup, doc_type: args[:doc_type]).clean
+        end
+      end
+    end
+
+    def segment
+      return [] unless text
+      case language
+      when 'en'
+        PragmaticSegmenter::Process.new(text: text, doc_type: doc_type).process
+      when 'de'
+        PragmaticSegmenter::Languages::Deutsch::Process.new(text: text, doc_type: doc_type).process
+      when 'es'
+        PragmaticSegmenter::Languages::Spanish::Process.new(text: text, doc_type: doc_type).process
+      when 'it'
+        PragmaticSegmenter::Languages::Italian::Process.new(text: text, doc_type: doc_type).process
+      when 'ja'
+        PragmaticSegmenter::Languages::Japanese::Process.new(text: text, doc_type: doc_type).process
+      when 'el'
+        PragmaticSegmenter::Languages::Greek::Process.new(text: text, doc_type: doc_type).process
+      when 'ru'
+        PragmaticSegmenter::Languages::Russian::Process.new(text: text, doc_type: doc_type).process
+      when 'ar'
+        PragmaticSegmenter::Languages::Arabic::Process.new(text: text, doc_type: doc_type).process
+      when 'am'
+        PragmaticSegmenter::Languages::Amharic::Process.new(text: text, doc_type: doc_type).process
+      when 'hi'
+        PragmaticSegmenter::Languages::Hindi::Process.new(text: text, doc_type: doc_type).process
+      when 'hy'
+        PragmaticSegmenter::Languages::Armenian::Process.new(text: text, doc_type: doc_type).process
+      when 'fa'
+        PragmaticSegmenter::Languages::Persian::Process.new(text: text, doc_type: doc_type).process
+      when 'my'
+        PragmaticSegmenter::Languages::Burmese::Process.new(text: text, doc_type: doc_type).process
+      when 'ur'
+        PragmaticSegmenter::Languages::Urdu::Process.new(text: text, doc_type: doc_type).process
+      else
+        PragmaticSegmenter::Process.new(text: text, doc_type: doc_type).process
+      end
+    end
+  end
+end

data/lib/pragmatic_segmenter/sentence_boundary_punctuation.rb

@@ -0,0 +1,17 @@
+# -*- encoding : utf-8 -*-
+
+module PragmaticSegmenter
+  # This class splits text at sentence boundary punctuation marks
+  class SentenceBoundaryPunctuation
+    SENTENCE_BOUNDARY_REGEX = /\u{ff08}(?:[^\u{ff09}])*\u{ff09}(?=\s?[A-Z])|\u{300c}(?:[^\u{300d}])*\u{300d}(?=\s[A-Z])|\((?:[^\)])*\)(?=\s[A-Z])|'(?:[^'])*'(?=\s[A-Z])|"(?:[^"])*"(?=\s[A-Z])|“(?:[^”])*”(?=\s[A-Z])|\S.*?[。．.！!?？ȸȹ☉☈☇☄]/
+
+    attr_reader :text
+    def initialize(text:)
+      @text = text
+    end
+
+    def split
+      text.scan(SENTENCE_BOUNDARY_REGEX)
+    end
+  end
+end

data/lib/pragmatic_segmenter/single_letter_abbreviation.rb

@@ -0,0 +1,37 @@
+# -*- encoding : utf-8 -*-
+
+module PragmaticSegmenter
+  # This class searches for periods within an abbreviation and
+  # replaces the periods.
+  class SingleLetterAbbreviation
+    # Rubular: http://rubular.com/r/e3H6kwnr6H
+    SingleUpperCaseLetterAtStartOfLineRule = Rule.new(/(?<=^[A-Z])\.(?=\s)/, '∯')
+
+    # Rubular: http://rubular.com/r/gitvf0YWH4
+    SingleUpperCaseLetterRule = Rule.new(/(?<=\s[A-Z])\.(?=\s)/, '∯')
+
+    attr_reader :text
+    def initialize(text:)
+      @text = text
+    end
+
+    def replace
+      @formatted_text = replace_single_letter_abbreviations(text)
+    end
+
+    private
+
+    def replace_single_letter_abbreviations(txt)
+      new_text = replace_single_uppercase_letter_abbreviation_at_start_of_line(txt)
+      replace_single_uppercase_letter_abbreviation(new_text)
+    end
+
+    def replace_single_uppercase_letter_abbreviation_at_start_of_line(txt)
+      txt.apply(SingleUpperCaseLetterAtStartOfLineRule)
+    end
+
+    def replace_single_uppercase_letter_abbreviation(txt)
+      txt.apply(SingleUpperCaseLetterRule)
+    end
+  end
+end

data/lib/pragmatic_segmenter/types.rb

@@ -0,0 +1,12 @@
+module PragmaticSegmenter
+  Rule = Struct.new(:pattern, :replacement)
+
+  class Text < String
+    def apply(*rules)
+      rules.flatten.each do |rule|
+        self.gsub!(rule.pattern, rule.replacement)
+      end
+      self
+    end
+  end
+end

data/lib/pragmatic_segmenter/version.rb

@@ -0,0 +1,3 @@
+module PragmaticSegmenter
+  VERSION = "0.0.1"
+end

data/pragmatic_segmenter.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pragmatic_segmenter/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pragmatic_segmenter"
+  spec.version       = PragmaticSegmenter::VERSION
+  spec.authors       = ["Kevin S. Dias"]
+  spec.email         = ["diasks2@gmail.com"]
+  spec.summary       = %q{A rule-based sentence boundary detection gem that works out-of-the-box across many languages}
+  spec.description   = %q{Pragmatic Segmenter is a sentence segmentation tool for Ruby. It allows you to split a text into an array of sentences. This gem provides 2 main benefits over other segmentation gems - 1) It works well even with ill-formatted text 2) It works for multiple languages }
+  spec.homepage      = "https://github.com/diasks2/pragmatic_segmenter"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rubocop"
+end