pragmatic_segmenter 0.3.1 → 0.3.2

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-02 00:00:00.000000000 Z
+date: 2015-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -70,17 +70,16 @@ files:
 - README.md
 - Rakefile
 - lib/pragmatic_segmenter.rb
-- lib/pragmatic_segmenter/abbreviation.rb
 - lib/pragmatic_segmenter/abbreviation_replacer.rb
 - lib/pragmatic_segmenter/between_punctuation.rb
 - lib/pragmatic_segmenter/cleaner.rb
-- lib/pragmatic_segmenter/ellipsis.rb
 - lib/pragmatic_segmenter/exclamation_words.rb
-- lib/pragmatic_segmenter/language_support.rb
+- lib/pragmatic_segmenter/languages.rb
 - lib/pragmatic_segmenter/languages/amharic.rb
 - lib/pragmatic_segmenter/languages/arabic.rb
 - lib/pragmatic_segmenter/languages/armenian.rb
 - lib/pragmatic_segmenter/languages/burmese.rb
+- lib/pragmatic_segmenter/languages/chinese.rb
 - lib/pragmatic_segmenter/languages/common.rb
 - lib/pragmatic_segmenter/languages/deutsch.rb
 - lib/pragmatic_segmenter/languages/dutch.rb
@@ -98,12 +97,11 @@ files:
 - lib/pragmatic_segmenter/list.rb
 - lib/pragmatic_segmenter/number.rb
 - lib/pragmatic_segmenter/process.rb
-- lib/pragmatic_segmenter/punctuation.rb
 - lib/pragmatic_segmenter/punctuation_replacer.rb
 - lib/pragmatic_segmenter/rules.rb
+- lib/pragmatic_segmenter/rules/ellipsis.rb
+- lib/pragmatic_segmenter/rules/html.rb
 - lib/pragmatic_segmenter/segmenter.rb
-- lib/pragmatic_segmenter/sentence_boundary_punctuation.rb
-- lib/pragmatic_segmenter/single_letter_abbreviation.rb
 - lib/pragmatic_segmenter/types.rb
 - lib/pragmatic_segmenter/version.rb
 - pragmatic_segmenter.gemspec

data/lib/pragmatic_segmenter/abbreviation.rb

@@ -1,22 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module PragmaticSegmenter
-  # Defines the abbreviations for each language (if available)
-  class Abbreviation
-    ABBREVIATIONS = ['adj', 'adm', 'adv', 'al', 'ala', 'alta', 'apr', 'arc', 'ariz', 'ark', 'art', 'assn', 'asst', 'attys', 'aug', 'ave', 'bart', 'bld', 'bldg', 'blvd', 'brig', 'bros', 'btw', 'cal', 'calif', 'capt', 'cl', 'cmdr', 'co', 'col', 'colo', 'comdr', 'con', 'conn', 'corp', 'cpl', 'cres', 'ct', 'd.phil', 'dak', 'dec', 'del', 'dept', 'det', 'dist', 'dr', 'dr.phil', 'dr.philos', 'drs', 'e.g', 'ens', 'esp', 'esq', 'etc', 'exp', 'expy', 'ext', 'feb', 'fed', 'fla', 'ft', 'fwy', 'fy', 'ga', 'gen', 'gov', 'hon', 'hosp', 'hr', 'hway', 'hwy', 'i.e', 'ia', 'id', 'ida', 'ill', 'inc', 'ind', 'ing', 'insp', 'is', 'jan', 'jr', 'jul', 'jun', 'kan', 'kans', 'ken', 'ky', 'la', 'lt', 'ltd', 'maj', 'man', 'mar', 'mass', 'may', 'md', 'me', 'med', 'messrs', 'mex', 'mfg', 'mich', 'min', 'minn', 'miss', 'mlle', 'mm', 'mme', 'mo', 'mont', 'mr', 'mrs', 'ms', 'msgr', 'mssrs', 'mt', 'mtn', 'neb', 'nebr', 'nev', 'no', 'nos', 'nov', 'nr', 'oct', 'ok', 'okla', 'ont', 'op', 'ord', 'ore', 'p', 'pa', 'pd', 'pde', 'penn', 'penna', 'pfc', 'ph', 'ph.d', 'pl', 'plz', 'pp', 'prof', 'pvt', 'que', 'rd', 'ref', 'rep', 'reps', 'res', 'rev', 'rt', 'sask', 'sen', 'sens', 'sep', 'sept', 'sfc', 'sgt', 'sr', 'st', 'supt', 'surg', 'tce', 'tenn', 'tex', 'univ', 'usafa', 'u.s', 'ut', 'va', 'v', 'ver', 'vs', 'vt', 'wash', 'wis', 'wisc', 'wy', 'wyo', 'yuk']
-    PREPOSITIVE_ABBREVIATIONS = ['adm', 'attys', 'brig', 'capt', 'cmdr', 'col', 'cpl', 'det', 'dr', 'gen', 'gov', 'ing', 'lt', 'maj', 'mr', 'mrs', 'ms', 'mt', 'messrs', 'mssrs', 'prof', 'ph', 'rep', 'reps', 'rev', 'sen', 'sens', 'sgt', 'st', 'supt', 'v', 'vs']
-    NUMBER_ABBREVIATIONS = ['art', 'ext', 'no', 'nos', 'p', 'pp']
-
-    def all
-      ABBREVIATIONS
-    end
-
-    def prepositive
-      PREPOSITIVE_ABBREVIATIONS
-    end
-
-    def number
-      NUMBER_ABBREVIATIONS
-    end
-  end
-end

data/lib/pragmatic_segmenter/language_support.rb

@@ -1,31 +0,0 @@
-module PragmaticSegmenter
-  module LanguageSupport
-    LANGUAGE_CODES = {
-      'en' => 'English',
-      'de' => 'Deutsch',
-      'es' => 'Spanish',
-      'fr' => 'French',
-      'it' => 'Italian',
-      'ja' => 'Japanese',
-      'el' => 'Greek',
-      'ru' => 'Russian',
-      'ar' => 'Arabic',
-      'am' => 'Amharic',
-      'hi' => 'Hindi',
-      'hy' => 'Armenian',
-      'fa' => 'Persian',
-      'my' => 'Burmese',
-      'ur' => 'Urdu',
-      'nl' => 'Dutch',
-      'pl' => 'Polish',
-    }
-
-    def process_class
-      Object.const_get("PragmaticSegmenter::Languages::#{LANGUAGE_CODES[language] || 'Common'}::Process")
-    end
-
-    def cleaner_class
-      Object.const_get("PragmaticSegmenter::Languages::#{LANGUAGE_CODES[language] || 'Common'}::Cleaner")
-    end
-  end
-end

data/lib/pragmatic_segmenter/punctuation.rb

@@ -1,12 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module PragmaticSegmenter
-  # This class holds the punctuation marks.
-  class Punctuation
-    PUNCT = ['。', '．', '.', '！', '!', '?', '？']
-
-    def punct
-      PUNCT
-    end
-  end
-end

data/lib/pragmatic_segmenter/sentence_boundary_punctuation.rb

@@ -1,17 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module PragmaticSegmenter
-  # This class splits text at sentence boundary punctuation marks
-  class SentenceBoundaryPunctuation
-    SENTENCE_BOUNDARY_REGEX = /\u{ff08}(?:[^\u{ff09}])*\u{ff09}(?=\s?[A-Z])|\u{300c}(?:[^\u{300d}])*\u{300d}(?=\s[A-Z])|\((?:[^\)]){2,}\)(?=\s[A-Z])|'(?:[^'])*[^,]'(?=\s[A-Z])|"(?:[^"])*[^,]"(?=\s[A-Z])|“(?:[^”])*[^,]”(?=\s[A-Z])|\S.*?[。．.！!?？ȸȹ☉☈☇☄]/
-
-    attr_reader :text
-    def initialize(text:)
-      @text = text
-    end
-
-    def split
-      text.scan(SENTENCE_BOUNDARY_REGEX)
-    end
-  end
-end

data/lib/pragmatic_segmenter/single_letter_abbreviation.rb

@@ -1,37 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module PragmaticSegmenter
-  # This class searches for periods within an abbreviation and
-  # replaces the periods.
-  class SingleLetterAbbreviation
-    # Rubular: http://rubular.com/r/e3H6kwnr6H
-    SingleUpperCaseLetterAtStartOfLineRule = Rule.new(/(?<=^[A-Z])\.(?=\s)/, '∯')
-
-    # Rubular: http://rubular.com/r/gitvf0YWH4
-    SingleUpperCaseLetterRule = Rule.new(/(?<=\s[A-Z])\.(?=\s)/, '∯')
-
-    attr_reader :text
-    def initialize(text:)
-      @text = text
-    end
-
-    def replace
-      @formatted_text = replace_single_letter_abbreviations(text)
-    end
-
-    private
-
-    def replace_single_letter_abbreviations(txt)
-      new_text = replace_single_uppercase_letter_abbreviation_at_start_of_line(txt)
-      replace_single_uppercase_letter_abbreviation(new_text)
-    end
-
-    def replace_single_uppercase_letter_abbreviation_at_start_of_line(txt)
-      txt.apply(SingleUpperCaseLetterAtStartOfLineRule)
-    end
-
-    def replace_single_uppercase_letter_abbreviation(txt)
-      txt.apply(SingleUpperCaseLetterRule)
-    end
-  end
-end