pragmatic_segmenter 0.3.3 → 0.3.4

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-27 00:00:00.000000000 Z
+date: 2015-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 'Pragmatic Segmenter is a sentence segmentation tool for Ruby. It allows
   you to split a text into an array of sentences. This gem provides 2 main benefits
   over other segmentation gems - 1) It works well even with ill-formatted text 2)
@@ -73,6 +87,7 @@ files:
 - lib/pragmatic_segmenter/abbreviation_replacer.rb
 - lib/pragmatic_segmenter/between_punctuation.rb
 - lib/pragmatic_segmenter/cleaner.rb
+- lib/pragmatic_segmenter/cleaner/rules.rb
 - lib/pragmatic_segmenter/exclamation_words.rb
 - lib/pragmatic_segmenter/languages.rb
 - lib/pragmatic_segmenter/languages/amharic.rb
@@ -81,6 +96,8 @@ files:
 - lib/pragmatic_segmenter/languages/burmese.rb
 - lib/pragmatic_segmenter/languages/chinese.rb
 - lib/pragmatic_segmenter/languages/common.rb
+- lib/pragmatic_segmenter/languages/common/ellipsis.rb
+- lib/pragmatic_segmenter/languages/common/numbers.rb
 - lib/pragmatic_segmenter/languages/deutsch.rb
 - lib/pragmatic_segmenter/languages/dutch.rb
 - lib/pragmatic_segmenter/languages/english.rb
@@ -95,17 +112,32 @@ files:
 - lib/pragmatic_segmenter/languages/spanish.rb
 - lib/pragmatic_segmenter/languages/urdu.rb
 - lib/pragmatic_segmenter/list.rb
-- lib/pragmatic_segmenter/number.rb
-- lib/pragmatic_segmenter/process.rb
+- lib/pragmatic_segmenter/processor.rb
 - lib/pragmatic_segmenter/punctuation_replacer.rb
-- lib/pragmatic_segmenter/rules.rb
-- lib/pragmatic_segmenter/rules/ellipsis.rb
-- lib/pragmatic_segmenter/rules/html.rb
 - lib/pragmatic_segmenter/segmenter.rb
 - lib/pragmatic_segmenter/types.rb
 - lib/pragmatic_segmenter/version.rb
 - pragmatic_segmenter.gemspec
 - spec/performance_spec.rb
+- spec/pragmatic_segmenter/languages/amharic_spec.rb
+- spec/pragmatic_segmenter/languages/arabic_spec.rb
+- spec/pragmatic_segmenter/languages/armenian_spec.rb
+- spec/pragmatic_segmenter/languages/burmese_spec.rb
+- spec/pragmatic_segmenter/languages/chinese_spec.rb
+- spec/pragmatic_segmenter/languages/deutsch_spec.rb
+- spec/pragmatic_segmenter/languages/dutch_spec.rb
+- spec/pragmatic_segmenter/languages/english_spec.rb
+- spec/pragmatic_segmenter/languages/french_spec.rb
+- spec/pragmatic_segmenter/languages/greek_spec.rb
+- spec/pragmatic_segmenter/languages/hindi_spec.rb
+- spec/pragmatic_segmenter/languages/italian_spec.rb
+- spec/pragmatic_segmenter/languages/japanese_spec.rb
+- spec/pragmatic_segmenter/languages/persian_spec.rb
+- spec/pragmatic_segmenter/languages/polish_spec.rb
+- spec/pragmatic_segmenter/languages/russian_spec.rb
+- spec/pragmatic_segmenter/languages/spanish_spec.rb
+- spec/pragmatic_segmenter/languages/urdu_spec.rb
+- spec/pragmatic_segmenter/languages_spec.rb
 - spec/pragmatic_segmenter_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/diasks2/pragmatic_segmenter
@@ -128,12 +160,31 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: A rule-based sentence boundary detection gem that works out-of-the-box across
   many languages
 test_files:
 - spec/performance_spec.rb
+- spec/pragmatic_segmenter/languages/amharic_spec.rb
+- spec/pragmatic_segmenter/languages/arabic_spec.rb
+- spec/pragmatic_segmenter/languages/armenian_spec.rb
+- spec/pragmatic_segmenter/languages/burmese_spec.rb
+- spec/pragmatic_segmenter/languages/chinese_spec.rb
+- spec/pragmatic_segmenter/languages/deutsch_spec.rb
+- spec/pragmatic_segmenter/languages/dutch_spec.rb
+- spec/pragmatic_segmenter/languages/english_spec.rb
+- spec/pragmatic_segmenter/languages/french_spec.rb
+- spec/pragmatic_segmenter/languages/greek_spec.rb
+- spec/pragmatic_segmenter/languages/hindi_spec.rb
+- spec/pragmatic_segmenter/languages/italian_spec.rb
+- spec/pragmatic_segmenter/languages/japanese_spec.rb
+- spec/pragmatic_segmenter/languages/persian_spec.rb
+- spec/pragmatic_segmenter/languages/polish_spec.rb
+- spec/pragmatic_segmenter/languages/russian_spec.rb
+- spec/pragmatic_segmenter/languages/spanish_spec.rb
+- spec/pragmatic_segmenter/languages/urdu_spec.rb
+- spec/pragmatic_segmenter/languages_spec.rb
 - spec/pragmatic_segmenter_spec.rb
 - spec/spec_helper.rb

data/lib/pragmatic_segmenter/number.rb

@@ -1,35 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module PragmaticSegmenter
-  # This class searches for numbers with periods within a string and
-  # replaces the periods.
-  class Number
-    # Rubular: http://rubular.com/r/oNyxBOqbyy
-    PeriodBeforeNumberRule = Rule.new(/\.(?=\d)/, '∯')
-
-    # Rubular: http://rubular.com/r/EMk5MpiUzt
-    NumberAfterPeriodBeforeLetterRule = Rule.new(/(?<=\d)\.(?=\S)/, '∯')
-
-    # Rubular: http://rubular.com/r/rf4l1HjtjG
-    NewLineNumberPeriodSpaceLetterRule = Rule.new(/(?<=\r\d)\.(?=(\s\S)|\))/, '∯')
-
-    # Rubular: http://rubular.com/r/HPa4sdc6b9
-    StartLineNumberPeriodRule = Rule.new(/(?<=^\d)\.(?=(\s\S)|\))/, '∯')
-
-    # Rubular: http://rubular.com/r/NuvWnKleFl
-    StartLineTwoDigitNumberPeriodRule = Rule.new(/(?<=^\d\d)\.(?=(\s\S)|\))/, '∯')
-
-    attr_reader :text
-    def initialize(text:)
-      @text = Text.new(text)
-    end
-
-    def replace
-      @text.apply PeriodBeforeNumberRule,
-        NumberAfterPeriodBeforeLetterRule,
-        NewLineNumberPeriodSpaceLetterRule,
-        StartLineNumberPeriodRule,
-        StartLineTwoDigitNumberPeriodRule
-    end
-  end
-end

data/lib/pragmatic_segmenter/rules.rb

@@ -1,168 +0,0 @@
-require 'pragmatic_segmenter/rules/html'
-
-module PragmaticSegmenter
-  module Rules
-
-    URL_EMAIL_KEYWORDS = ['@', 'http', '.com', 'net', 'www', '//']
-
-    # Rubular: http://rubular.com/r/6dt98uI76u
-    NO_SPACE_BETWEEN_SENTENCES_REGEX = /(?<=[a-z])\.(?=[A-Z])/
-
-    # Rubular: http://rubular.com/r/l6KN6rH5XE
-    NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX = /(?<=\d)\.(?=[A-Z])/
-
-    # Rubular: http://rubular.com/r/V57WnM9Zut
-    NewLineInMiddleOfWordRule = Rule.new(/\n(?=[a-zA-Z]{1,2}\n)/, '')
-
-    # Rubular: http://rubular.com/r/3GiRiP2IbD
-    NEWLINE_IN_MIDDLE_OF_SENTENCE_REGEX = /(?<=\s)\n(?=([a-z]|\())/
-
-    # Rubular: http://rubular.com/r/UZAVcwqck8
-    PDF_NewLineInMiddleOfSentenceRule = Rule.new(/(?<=[^\n]\s)\n(?=\S)/, '')
-
-    # Rubular: http://rubular.com/r/eaNwGavmdo
-    PDF_NewLineInMiddleOfSentenceNoSpacesRule = Rule.new(/\n(?=[a-z])/, ' ')
-
-    # Rubular: http://rubular.com/r/bAJrhyLNeZ
-    InlineFormattingRule = Rule.new(/\{b\^&gt;\d*&lt;b\^\}|\{b\^>\d*<b\^\}/, '')
-
-    # Rubular: http://rubular.com/r/dMxp5MixFS
-    DoubleNewLineWithSpaceRule = Rule.new(/\n \n/, "\r")
-
-    # Rubular: http://rubular.com/r/H6HOJeA8bq
-    DoubleNewLineRule = Rule.new(/\n\n/, "\r")
-
-    # Rubular: http://rubular.com/r/Gn18aAnLdZ
-    NewLineFollowedByBulletRule = Rule.new(/\n(?=•)/, "\r")
-
-    # Rubular: http://rubular.com/r/FseyMiiYFT
-    NewLineFollowedByPeriodRule = Rule.new(/\n(?=\.(\s|\n))/, '')
-
-    # Rubular: http://rubular.com/r/8mc1ArOIGy
-    TableOfContentsRule = Rule.new(/\.{5,}\s*\d+-*\d*/, "\r")
-
-    # Rubular: http://rubular.com/r/DwNSuZrNtk
-    ConsecutivePeriodsRule = Rule.new(/\.{5,}/, ' ')
-
-    # Rubular: http://rubular.com/r/IQ4TPfsbd8
-    ConsecutiveForwardSlashRule = Rule.new(/\/{3}/, '')
-
-    # Rubular: http://rubular.com/r/6dt98uI76u
-    NoSpaceBetweenSentencesRule = Rule.new(NO_SPACE_BETWEEN_SENTENCES_REGEX, '. ')
-
-    # Rubular: http://rubular.com/r/l6KN6rH5XE
-    NoSpaceBetweenSentencesDigitRule = Rule.new(NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX, '. ')
-
-    EscapedCarriageReturnRule = Rule.new(/\\r/, "\r")
-    TypoEscapedCarriageReturnRule = Rule.new(/\\\ r/, "\r")
-
-    EscapedNewLineRule = Rule.new(/\\n/, "\n")
-    TypoEscapedNewLineRule = Rule.new(/\\\ n/, "\n")
-
-    ReplaceNewlineWithCarriageReturnRule = Rule.new(/\n/, "\r")
-
-    QuotationsFirstRule = Rule.new(/''/, '"')
-    QuotationsSecondRule = Rule.new(/``/, '"')
-
-    # Rubular: http://rubular.com/r/EUbZCNfgei
-    AbbreviationsWithMultiplePeriodsAndEmailRule = Rule.new(/(\w)(\.)(\w)/, '\1∮\3')
-
-    # Rubular: http://rubular.com/r/G2opjedIm9
-    GeoLocationRule = Rule.new(/(?<=[a-zA-z]°)\.(?=\s*\d+)/, '∯')
-
-    SingleNewLineRule = Rule.new(/\n/, 'ȹ')
-
-    SubSingleQuoteRule = Rule.new(/&⎋&/, "'")
-
-    ExtraWhiteSpaceRule = Rule.new(/\s{3,}/, ' ')
-
-    # Rubular: http://rubular.com/r/aXPUGm6fQh
-    QuestionMarkInQuotationRule = Rule.new(/\?(?=(\'|\"))/, '&ᓷ&')
-
-    module ExclamationPointRules
-      # Rubular: http://rubular.com/r/XS1XXFRfM2
-      InQuotationRule = Rule.new(/\!(?=(\'|\"))/, '&ᓴ&')
-
-      # Rubular: http://rubular.com/r/sl57YI8LkA
-      BeforeCommaMidSentenceRule = Rule.new(/\!(?=\,\s[a-z])/, '&ᓴ&')
-
-      # Rubular: http://rubular.com/r/f9zTjmkIPb
-      MidSentenceRule = Rule.new(/\!(?=\s[a-z])/, '&ᓴ&')
-
-      All = [ InQuotationRule, BeforeCommaMidSentenceRule, MidSentenceRule ]
-    end
-
-    module DoublePunctuationRules
-      FirstRule = Rule.new(/\?!/, '☉')
-      SecondRule = Rule.new(/!\?/, '☈')
-      ThirdRule = Rule.new(/\?\?/, '☇')
-      ForthRule = Rule.new(/!!/, '☄')
-
-      All = [ FirstRule, SecondRule, ThirdRule, ForthRule ]
-    end
-
-    module ReinsertEllipsisRules
-      SubThreeConsecutivePeriod = Rule.new(/ƪ/, '...')
-      SubThreeSpacePeriod = Rule.new(/♟/, ' . . . ')
-      SubFourSpacePeriod = Rule.new(/♝/, '. . . .')
-      SubTwoConsecutivePeriod = Rule.new(/☏/, '..')
-      SubOnePeriod = Rule.new(/∮/, '.')
-
-      All = [ SubThreeConsecutivePeriod, SubThreeSpacePeriod,
-              SubFourSpacePeriod, SubTwoConsecutivePeriod,
-              SubOnePeriod ]
-    end
-
-    module SubSymbolsRules
-      Period = Rule.new(/∯/, '.')
-      ArabicComma = Rule.new(/♬/, '،')
-      SemiColon = Rule.new(/♭/, ':')
-      FullWidthPeriod = Rule.new(/&ᓰ&/, '。')
-      SpecialPeriod = Rule.new(/&ᓱ&/, '．')
-      FullWidthExclamation = Rule.new(/&ᓳ&/, '！')
-      ExclamationPoint = Rule.new(/&ᓴ&/, '!')
-      QuestionMark = Rule.new(/&ᓷ&/, '?')
-      FullWidthQuestionMark = Rule.new(/&ᓸ&/, '？')
-      MixedDoubleQE = Rule.new(/☉/, '?!')
-      MixedDoubleQQ = Rule.new(/☇/, '??')
-      MixedDoubleEQ = Rule.new(/☈/, '!?')
-      MixedDoubleEE = Rule.new(/☄/, '!!')
-      LeftParens = Rule.new(/&✂&/, '(')
-      RightParens = Rule.new(/&⌬&/, ')')
-      TemporaryEndingPunctutation = Rule.new('ȸ', '')
-      Newline = Rule.new(/ȹ/, "\n")
-
-      All = [ Period, ArabicComma,
-              SemiColon, FullWidthPeriod,
-              SpecialPeriod, FullWidthExclamation,
-              ExclamationPoint, QuestionMark,
-              FullWidthQuestionMark, MixedDoubleQE,
-              MixedDoubleQQ, MixedDoubleEQ,
-              MixedDoubleEE, LeftParens,
-              RightParens, TemporaryEndingPunctutation,
-              Newline ]
-    end
-
-    module EscapeRegexReservedCharacters
-      LeftParen = Rule.new('(', '\\(')
-      RightParen = Rule.new(')', '\\)')
-      LeftBracket = Rule.new('[', '\\[')
-      RightBracket = Rule.new(']', '\\]')
-      Dash = Rule.new('-', '\\-')
-
-      All = [ LeftParen, RightParen,
-              LeftBracket, RightBracket, Dash ]
-    end
-
-    module SubEscapedRegexReservedCharacters
-      SubLeftParen = Rule.new('\\(', '(')
-      SubRightParen = Rule.new('\\)', ')')
-      SubLeftBracket = Rule.new('\\[', '[')
-      SubRightBracket = Rule.new('\\]', ']')
-      SubDash = Rule.new('\\-', '-')
-
-      All = [ SubLeftParen, SubRightParen,
-              SubLeftBracket, SubRightBracket, SubDash ]
-    end
-  end
-end

data/lib/pragmatic_segmenter/rules/ellipsis.rb

@@ -1,35 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module PragmaticSegmenter
-  module Rules
-    # This class searches for ellipses within a string and
-    # replaces the periods.
-
-    # http://www.dailywritingtips.com/in-search-of-a-4-dot-ellipsis/
-    # http://www.thepunctuationguide.com/ellipses.html
-
-    module EllipsisRules
-      # Rubular: http://rubular.com/r/i60hCK81fz
-      ThreeConsecutiveRule = Rule.new(/\.\.\.(?=\s+[A-Z])/, '☏.')
-
-      # Rubular: http://rubular.com/r/Hdqpd90owl
-      FourConsecutiveRule = Rule.new(/(?<=\S)\.{3}(?=\.\s[A-Z])/, 'ƪ')
-
-      # Rubular: http://rubular.com/r/YBG1dIHTRu
-      ThreeSpaceRule = Rule.new(/(\s\.){3}\s/, '♟')
-
-      # Rubular: http://rubular.com/r/2VvZ8wRbd8
-      FourSpaceRule = Rule.new(/(?<=[a-z])(\.\s){3}\.(\z|$|\n)/, '♝')
-
-      OtherThreePeriodRule = Rule.new(/\.\.\./, 'ƪ')
-
-      All = [
-        ThreeSpaceRule,
-        FourSpaceRule,
-        FourConsecutiveRule,
-        ThreeConsecutiveRule,
-        OtherThreePeriodRule
-      ]
-    end
-  end
-end

data/lib/pragmatic_segmenter/rules/html.rb

@@ -1,13 +0,0 @@
-module PragmaticSegmenter
-  module Rules
-    module HTMLRules
-      # Rubular: http://rubular.com/r/ENrVFMdJ8v
-      HTMLTagRule = Rule.new(/<\/?[^>]*>/, '')
-
-      # Rubular: http://rubular.com/r/XZVqMPJhea
-      EscapedHTMLTagRule = Rule.new(/&lt;\/?[^gt;]*gt;/, '')
-
-      All = [HTMLTagRule, EscapedHTMLTagRule]
-    end
-  end
-end