pragmatic_segmenter 0.3.17 → 0.3.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: baf0f9cb38a40398530e15df0d28ab8a654c49c6
-  data.tar.gz: 4a184ffa70092a3f99fe7d194a71e7a935b1d3b2
+  metadata.gz: 3bb581e56e988521adc41dbb94bc7281ee7dfa95
+  data.tar.gz: 0c3b6fe877a5d39d36053b7ed68a860b5d17779b
 SHA512:
-  metadata.gz: 089a40744464256d33ce30b4c6d40a6f00bcd5dd0178757a3d65b3f5ff18242751e5074902bd0aab0460e97e5db0040a30f206648e98c14ec861abf0ba92680c
-  data.tar.gz: e49bdbe345e2d2e394d2f4ef03eac2ca0488f3dac158dfd5c193a328368326963785bf643a565d599d0e4d49196e3480d7f8054ec1df4927c192958bb1e1de47
+  metadata.gz: 1b1dd64b5a382e8bb7ed5d79fbb9264565d71088f234ba4a9bd7cae47184e1c64f78b32a72f39326aa31936c6c6742aa2ff75cd75cd1b328987a6061a4d2534b
+  data.tar.gz: c150b178c93b7183300559e89c117cf8f9f93adf6ef33790a3ce0b292c66588fe6461c724210f7f93e078d2d08a10e995d7234bad963f8d5aa1c52c378effe5e

data/NEWS

@@ -1,3 +1,7 @@
+0.3.18 (2018-03-27):
+
+* Improvement: Performance optimizations
+
 0.3.17 (2017-12-07):
 
 * Bug Fix: Regex for parsing HTML

data/README.md

@@ -77,7 +77,7 @@ Pragmatic Segmenter is opinionated and made for the explicit purpose of segmenti
 Pragmatic Segmenter is specifically used for the purpose of segmenting texts for use in translation (and translation memory) related applications. Therefore Pragmatic Segmenter takes a stance on some formatting and segmentation gray areas with the goal of improving the segmentation for the above stated purpose. Some examples:
 
 - Removes 'table of contents' style long string of periods ('............')
-- Keeps parenthetical sentences within a sentence as one segment for clarity even though technically there are multiple grammatical sentences within the segment
+- Keeps parentheticals, quotations, and parentheticals or quotations within a sentence as one segment for clarity even though technically there may be multiple grammatical sentences within the segment
 - Strips out any xhtml code
 - Conservative in cases where the sentence boundary is ambigious and Pragmatic Segmenter does not have a built in rule
 
@@ -862,6 +862,9 @@ To test the relative performance of different segmentation tools and libraries I
 **Version 0.3.17**
 * Fix issue involving the HTML regex in the cleaner
 
+**Version 0.3.18**
+* Performance optimizations
+
 ## Contributing
 
 If you find a text that is incorrectly segmented using this gem, please submit an issue.

data/lib/pragmatic_segmenter/abbreviation_replacer.rb

@@ -1,4 +1,6 @@
 # -*- encoding : utf-8 -*-
+# frozen_string_literal: true
+
 require 'unicode'
 
 module PragmaticSegmenter
@@ -28,11 +30,12 @@ module PragmaticSegmenter
     def search_for_abbreviations_in_string(txt)
       original = txt.dup
       downcased = Unicode::downcase(txt)
-      @language::Abbreviation::ABBREVIATIONS.each do |a|
-        next unless downcased.include?(a.strip)
-        abbrev_match = original.scan(/(?:^|\s|\r|\n)#{Regexp.escape(a.strip)}/i)
+      @language::Abbreviation::ABBREVIATIONS.each do |abbreviation|
+        stripped = abbreviation.strip
+        next unless downcased.include?(stripped)
+        abbrev_match = original.scan(/(?:^|\s|\r|\n)#{Regexp.escape(stripped)}/i)
         next if abbrev_match.empty?
-        next_word_start = /(?<=#{Regexp.escape(a.strip)} ).{1}/
+        next_word_start = /(?<=#{Regexp.escape(stripped)} ).{1}/
         character_array = @text.scan(next_word_start)
         abbrev_match.each_with_index do |am, index|
           txt = scan_for_replacements(txt, am, index, character_array)
@@ -74,19 +77,11 @@ module PragmaticSegmenter
       # and try to cover the words that most often start a
       # sentence but could never follow one of the abbreviations below.
 
+      # Rubular: http://rubular.com/r/PkBQ3PVBS8
       @language::AbbreviationReplacer::SENTENCE_STARTERS.each do |word|
         escaped = Regexp.escape(word)
-        txt.gsub!(/U∯S∯\s#{escaped}\s/, "U∯S\.\s#{escaped}\s")
-        txt.gsub!(/U\.S∯\s#{escaped}\s/, "U\.S\.\s#{escaped}\s")
-        txt.gsub!(/U∯K∯\s#{escaped}\s/, "U∯K\.\s#{escaped}\s")
-        txt.gsub!(/U\.K∯\s#{escaped}\s/, "U\.K\.\s#{escaped}\s")
-        txt.gsub!(/E∯U∯\s#{escaped}\s/, "E∯U\.\s#{escaped}\s")
-        txt.gsub!(/E\.U∯\s#{escaped}\s/, "E\.U\.\s#{escaped}\s")
-        txt.gsub!(/U∯S∯A∯\s#{escaped}\s/, "U∯S∯A\.\s#{escaped}\s")
-        txt.gsub!(/U\.S\.A∯\s#{escaped}\s/, "U\.S\.A\.\s#{escaped}\s")
-        txt.gsub!(/I∯\s#{escaped}\s/, "I\.\s#{escaped}\s")
-        txt.gsub!(/i.v∯\s#{escaped}\s/, "i\.v\.\s#{escaped}\s")
-        txt.gsub!(/I.V∯\s#{escaped}\s/, "I\.V\.\s#{escaped}\s")
+        regex   = /(U∯S|U\.S|U∯K|E∯U|E\.U|U∯S∯A|U\.S\.A|I|i.v|I.V)∯(?=\s#{escaped}\s)/
+        txt.gsub!(regex, '\1.')
       end
       txt
     end

data/lib/pragmatic_segmenter/between_punctuation.rb

@@ -1,4 +1,5 @@
 # -*- encoding : utf-8 -*-
+# frozen_string_literal: true
 
 module PragmaticSegmenter
   # This class searches for punctuation between quotes or parenthesis

data/lib/pragmatic_segmenter/cleaner.rb

@@ -1,4 +1,6 @@
 # -*- encoding : utf-8 -*-
+# frozen_string_literal: true
+
 require_relative 'cleaner/rules'
 
 module PragmaticSegmenter
@@ -62,7 +64,7 @@ module PragmaticSegmenter
 
     def replace_punctuation_in_brackets
       @text.dup.gsub!(/\[(?:[^\]])*\]/) do |match|
-        @text.gsub!(/#{Regexp.escape(match)}/, "#{match.dup.gsub!(/\?/, '&ᓷ&')}") if match.include?('?')
+        @text.gsub!(/#{Regexp.escape(match)}/, match.dup.gsub!(/\?/, '&ᓷ&')) if match.include?('?')
       end
     end
 

data/lib/pragmatic_segmenter/cleaner/rules.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   # This is an opinionated class that removes errant newlines,
   # xhtml, inline formatting, etc.

data/lib/pragmatic_segmenter/exclamation_words.rb

@@ -1,19 +1,20 @@
 # -*- encoding : utf-8 -*-
+# frozen_string_literal: true
+
 require 'pragmatic_segmenter/punctuation_replacer'
 
 module PragmaticSegmenter
   # This class searches for exclamation points that
   # are part of words and not ending punctuation and replaces them.
   module ExclamationWords
-    WORDS_WITH_EXCLAMATIONS = ['!Xũ', '!Kung', 'ǃʼOǃKung', '!Xuun', '!Kung-Ekoka', 'ǃHu', 'ǃKhung', 'ǃKu', 'ǃung', 'ǃXo', 'ǃXû', 'ǃXung', 'ǃXũ', '!Xun', 'Yahoo!', 'Y!J', 'Yum!']
+    EXCLAMATION_WORDS = %w[!Xũ !Kung ǃʼOǃKung !Xuun !Kung-Ekoka ǃHu ǃKhung ǃKu ǃung ǃXo ǃXû ǃXung ǃXũ !Xun Yahoo! Y!J Yum!].freeze
+    REGEXP            = Regexp.new(EXCLAMATION_WORDS.map { |string| Regexp.escape(string) }.join('|'))
 
     def self.apply_rules(text)
-      WORDS_WITH_EXCLAMATIONS.each do |exclamation|
-        PragmaticSegmenter::PunctuationReplacer.new(
-          matches_array: text.scan(/#{Regexp.escape(exclamation)}/),
-          text: text
-        ).replace
-      end
+      PragmaticSegmenter::PunctuationReplacer.new(
+        matches_array: text.scan(REGEXP),
+        text: text
+      ).replace
     end
   end
 end

data/lib/pragmatic_segmenter/languages.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'pragmatic_segmenter/types'
 require 'pragmatic_segmenter/processor'
 require 'pragmatic_segmenter/cleaner'

data/lib/pragmatic_segmenter/languages/amharic.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Amharic

data/lib/pragmatic_segmenter/languages/arabic.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Arabic

data/lib/pragmatic_segmenter/languages/armenian.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Armenian

data/lib/pragmatic_segmenter/languages/bulgarian.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Bulgarian

data/lib/pragmatic_segmenter/languages/burmese.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Burmese

data/lib/pragmatic_segmenter/languages/chinese.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Chinese

data/lib/pragmatic_segmenter/languages/common.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'common/numbers'
 require_relative 'common/ellipsis'
 

data/lib/pragmatic_segmenter/languages/common/ellipsis.rb

@@ -1,4 +1,5 @@
 # -*- encoding : utf-8 -*-
+# frozen_string_literal: true
 
 module PragmaticSegmenter
   module Languages

data/lib/pragmatic_segmenter/languages/common/numbers.rb

@@ -1,4 +1,5 @@
 # -*- encoding : utf-8 -*-
+# frozen_string_literal: true
 
 module PragmaticSegmenter
   module Languages

data/lib/pragmatic_segmenter/languages/danish.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Danish

data/lib/pragmatic_segmenter/languages/deutsch.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Deutsch

data/lib/pragmatic_segmenter/languages/dutch.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Dutch

data/lib/pragmatic_segmenter/languages/english.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module English

data/lib/pragmatic_segmenter/languages/french.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module French

data/lib/pragmatic_segmenter/languages/greek.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Greek

data/lib/pragmatic_segmenter/languages/hindi.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Hindi

data/lib/pragmatic_segmenter/languages/italian.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Italian

data/lib/pragmatic_segmenter/languages/japanese.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Japanese

data/lib/pragmatic_segmenter/languages/persian.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Persian

data/lib/pragmatic_segmenter/languages/polish.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Polish

data/lib/pragmatic_segmenter/languages/russian.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Russian

data/lib/pragmatic_segmenter/languages/spanish.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Spanish

data/lib/pragmatic_segmenter/languages/urdu.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   module Languages
     module Urdu

data/lib/pragmatic_segmenter/list.rb

@@ -1,4 +1,5 @@
 # -*- encoding : utf-8 -*-
+# frozen_string_literal: true
 
 module PragmaticSegmenter
   # This class searches for a list within a string and adds
@@ -41,6 +42,10 @@ module PragmaticSegmenter
     ALPHABETICAL_LIST_LETTERS_AND_PERIODS_REGEX =
       /(?<=^)[a-z]\.|(?<=\A)[a-z]\.|(?<=\s)[a-z]\./i
 
+    # Rubular: http://rubular.com/r/GcnmQt4a3I
+    ROMAN_NUMERALS_IN_PARENTHESES =
+        /\(((?=[mdclxvi])m*(c[md]|d?c*)(x[cl]|l?x*)(i[xv]|v?i*))\)(?=\s[A-Z])/
+
     attr_reader :text
     def initialize(text:)
       @text = Text.new(text)
@@ -54,12 +59,7 @@ module PragmaticSegmenter
     end
 
     def replace_parens
-      ROMAN_NUMERALS.each do |rm|
-        next unless text =~ /\(#{Regexp.escape(rm)}\)\s[A-Z]/
-        text.gsub!(/\(#{Regexp.escape(rm)}\)(?=\s[A-Z])/) do |match|
-          match.gsub!(/\(/, '&✂&').gsub!(/\)/, '&⌬&')
-        end
-      end
+      text.gsub!(ROMAN_NUMERALS_IN_PARENTHESES, '&✂&\1&⌬&'.freeze)
       text
     end
 

data/lib/pragmatic_segmenter/processor.rb

@@ -1,4 +1,6 @@
 # -*- encoding : utf-8 -*-
+# frozen_string_literal: true
+
 require 'pragmatic_segmenter/punctuation_replacer'
 require 'pragmatic_segmenter/between_punctuation'
 

data/lib/pragmatic_segmenter/punctuation_replacer.rb

@@ -1,4 +1,5 @@
 # -*- encoding : utf-8 -*-
+# frozen_string_literal: true
 
 module PragmaticSegmenter
   # This class replaces punctuation that is typically a sentence boundary
@@ -63,7 +64,7 @@ module PragmaticSegmenter
 
     def sub_characters(string, char_a, char_b)
       sub = string.gsub(char_a, char_b)
-      @text.gsub!(/#{Regexp.escape(string)}/, "#{sub}")
+      @text.gsub!(/#{Regexp.escape(string)}/, sub)
       sub
     end
   end

data/lib/pragmatic_segmenter/segmenter.rb

@@ -1,4 +1,6 @@
 # -*- encoding : utf-8 -*-
+# frozen_string_literal: true
+
 require 'pragmatic_segmenter/languages'
 
 module PragmaticSegmenter

data/lib/pragmatic_segmenter/types.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
   Rule = Struct.new(:pattern, :replacement)
 

data/lib/pragmatic_segmenter/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PragmaticSegmenter
-  VERSION = "0.3.17"
+  VERSION = "0.3.18"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.3.17
+  version: 0.3.18
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-07 00:00:00.000000000 Z
+date: 2018-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode
@@ -180,7 +180,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.4.1
 signing_key: 
 specification_version: 4
 summary: A rule-based sentence boundary detection gem that works out-of-the-box across