pragmatic_segmenter 0.3.22 → 0.3.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 77e3c8da3e5e184a5143dff774dbe851de58c385
-  data.tar.gz: f9f2803fb49a298b2015e206f7237b792be26e10
+SHA256:
+  metadata.gz: 51ae71a6650fcd15671ac767d26ebe1315a9ea655d8fbf6e29ef9e4fa668fc93
+  data.tar.gz: 786246dc9e80872b423013fed2d69e0cba48cc7a7d5a693a3165b4cdf61fe00d
 SHA512:
-  metadata.gz: 050ea08ad001c6786f44c936581e4d82d05ebf2cc1ac5265a5b24c3af25d8cad4f562f3d7c241baf48d49a14374559d12e4888f7514f9edb23733645c49999d7
-  data.tar.gz: d5c39d307836ce2571f8c8b0507110dd907d5150a6efc686136d915e2ecef6bca43a262ac7fc76fa908a9f18e9a1344f59b4da9115806ddc6e24e536bea0bd7d
+  metadata.gz: a830c5787a3818bc274b69aabd82bf5f837ba76c43921970c26a59f229d69bb24b698ff27389056ed6c536216edefdf4fa12338affbe883929b492065554af4c
+  data.tar.gz: f86cd6a66eaeb1890b5ddb2316d5ede734061b78212a490f8092bd20845cdb4dd47fac374972244785170ae266af21c566cbf59dd1a5667151ccd651269b72d8

data/NEWS

@@ -1,3 +1,12 @@
+0.3.24 (2024-08-12):
+
+* Bug Fix: Catastrophic backtracking in regular expression for numerical references
+* Improvement: Remove unicode dependency
+
+0.3.23 (2021-05-03):
+
+* Improvement: Refactor for Ruby 3.0 compatibility
+
 0.3.22 (2018-09-23):
 
 * Improvement: Initial support for Kazakh

data/README.md

@@ -1,6 +1,6 @@
 # Pragmatic Segmenter
 
-[![Gem Version](https://badge.fury.io/rb/pragmatic_segmenter.svg)](http://badge.fury.io/rb/pragmatic_segmenter) [![Code Climate](https://codeclimate.com/github/diasks2/pragmatic_segmenter/badges/gpa.svg)](https://codeclimate.com/github/diasks2/pragmatic_segmenter) [![Build Status](https://travis-ci.org/diasks2/pragmatic_segmenter.png)](https://travis-ci.org/diasks2/pragmatic_segmenter) [![Test Coverage](https://codeclimate.com/github/diasks2/pragmatic_segmenter/badges/coverage.svg)](https://codeclimate.com/github/diasks2/pragmatic_segmenter) [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/diasks2/pragmatic_segmenter/blob/master/LICENSE.txt)
+[![Gem Version](https://badge.fury.io/rb/pragmatic_segmenter.svg)](http://badge.fury.io/rb/pragmatic_segmenter) [![Build Status](https://travis-ci.org/diasks2/pragmatic_segmenter.png)](https://travis-ci.org/diasks2/pragmatic_segmenter) [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/diasks2/pragmatic_segmenter/blob/master/LICENSE.txt)
 
 Pragmatic Segmenter is a rule-based sentence boundary detection gem that works out-of-the-box across many languages.
 
@@ -887,6 +887,13 @@ To test the relative performance of different segmentation tools and libraries I
 **Version 0.3.22**
 * Add initial support and tests for Kazakh
 
+**Version 0.3.23**
+* Refactor for Ruby 3.0 compatibility
+
+**Version 0.3.24**
+* Fix catastrophic backtracking in regular expression for numerical references
+* Remove unicode dependency
+
 ## Contributing
 
 If you find a text that is incorrectly segmented using this gem, please submit an issue.
@@ -900,6 +907,7 @@ If you find a text that is incorrectly segmented using this gem, please submit a
 ## Ports
 
 * [C# - PragmaticSegmenterNet](https://github.com/UglyToad/PragmaticSegmenterNet)
+* [Python - pySBD](https://github.com/nipunsadvilkar/pySBD)
 
 ## License
 

data/lib/pragmatic_segmenter/abbreviation_replacer.rb

@@ -10,18 +10,19 @@ module PragmaticSegmenter
 
     attr_reader :text
     def initialize(text:, language: )
-      @text = Text.new(text)
+      @text = text.dup
       @language = language
     end
 
     def replace
-      @text.apply(@language::PossessiveAbbreviationRule,
+      Rule.apply(@text,
+        @language::PossessiveAbbreviationRule,
         @language::KommanditgesellschaftRule,
         @language::SingleLetterAbbreviationRules::All)
 
       @text = search_for_abbreviations_in_string(@text)
       @text = replace_multi_period_abbreviations(@text)
-      @text.apply(@language::AmPmRules::All)
+      Rule.apply(@text, @language::AmPmRules::All)
       replace_abbreviation_as_sentence_boundary(@text)
     end
 

data/lib/pragmatic_segmenter/cleaner.rb

@@ -11,7 +11,7 @@ module PragmaticSegmenter
 
     attr_reader :text, :doc_type
     def initialize(text:, doc_type: nil, language: Languages::Common)
-      @text = Text.new(text)
+      @text = text.dup
       @doc_type = doc_type
       @language = language
     end
@@ -37,10 +37,10 @@ module PragmaticSegmenter
       replace_newlines
       replace_escaped_newlines
 
-      @text.apply(HTML::All)
+      Rule.apply(@text, HTML::All)
 
       replace_punctuation_in_brackets
-      @text.apply(InlineFormattingRule)
+      Rule.apply(@text, InlineFormattingRule)
       clean_quotations
       clean_table_of_contents
       check_for_no_space_in_between_sentences
@@ -72,7 +72,7 @@ module PragmaticSegmenter
       if word =~ regex
         unless URL_EMAIL_KEYWORDS.any? { |web| word =~ /#{web}/ }
           unless abbreviations.any? { |abbr| word =~ /#{abbr}/i }
-            new_word = word.dup.apply(rule)
+            new_word = Rule.apply(word.dup, rule)
             txt.gsub!(/#{Regexp.escape(word)}/, new_word)
           end
         end
@@ -92,45 +92,45 @@ module PragmaticSegmenter
     end
 
     def remove_newline_in_middle_of_word
-      @text.apply NewLineInMiddleOfWordRule
+      Rule.apply @text, NewLineInMiddleOfWordRule
     end
 
     def replace_escaped_newlines
-      @text.apply EscapedNewLineRule, EscapedCarriageReturnRule,
+      Rule.apply @text, EscapedNewLineRule, EscapedCarriageReturnRule,
         TypoEscapedNewLineRule, TypoEscapedCarriageReturnRule
     end
 
     def replace_double_newlines
-      @text.apply DoubleNewLineWithSpaceRule, DoubleNewLineRule
+      Rule.apply @text, DoubleNewLineWithSpaceRule, DoubleNewLineRule
     end
 
     def replace_newlines
       if doc_type.eql?('pdf')
         remove_pdf_line_breaks
       else
-        @text.apply NewLineFollowedByPeriodRule,
+        Rule.apply @text, NewLineFollowedByPeriodRule,
           ReplaceNewlineWithCarriageReturnRule
       end
     end
 
     def remove_pdf_line_breaks
-      @text.apply NewLineFollowedByBulletRule,
+      Rule.apply @text, NewLineFollowedByBulletRule,
 
         PDF::NewLineInMiddleOfSentenceRule,
         PDF::NewLineInMiddleOfSentenceNoSpacesRule
     end
 
     def clean_quotations
-      @text.apply QuotationsFirstRule, QuotationsSecondRule
+      Rule.apply @text, QuotationsFirstRule, QuotationsSecondRule
     end
 
     def clean_table_of_contents
-      @text.apply TableOfContentsRule, ConsecutivePeriodsRule,
+      Rule.apply @text, TableOfContentsRule, ConsecutivePeriodsRule,
         ConsecutiveForwardSlashRule
     end
 
     def clean_consecutive_characters
-      @text.apply ConsecutivePeriodsRule, ConsecutiveForwardSlashRule
+      Rule.apply @text, ConsecutivePeriodsRule, ConsecutiveForwardSlashRule
     end
   end
 end

data/lib/pragmatic_segmenter/languages/common/numbers.rb

@@ -47,7 +47,7 @@ module PragmaticSegmenter
       # Rubular: http://rubular.com/r/mQ8Es9bxtk
       CONTINUOUS_PUNCTUATION_REGEX = /(?<=\S)(!|\?){3,}(?=(\s|\z|$))/
 
-      NUMBERED_REFERENCE_REGEX = /(?<=[^\d\s])(\.|∯)((\[(\d{1,3},?\s?-?\s?)*\b\d{1,3}\])+|((\d{1,3}\s?)*\d{1,3}))(\s)(?=[A-Z])/
+      NUMBERED_REFERENCE_REGEX = /(?<=[^\d\s])(\.|∯)((\[(\d{1,3},?\s?-?\s?)?\b\d{1,3}\])+|((\d{1,3}\s?){0,3}\d{1,3}))(\s)(?=[A-Z])/
 
       # Rubular: http://rubular.com/r/yqa4Rit8EY
       PossessiveAbbreviationRule = Rule.new(/\.(?='s\s)|\.(?='s$)|\.(?='s\z)/, '∯')

data/lib/pragmatic_segmenter/languages/deutsch.rb

@@ -47,7 +47,7 @@ module PragmaticSegmenter
         private
 
         def replace_numbers
-          @text.apply Numbers::All
+          Rule.apply @text, Numbers::All
 
           replace_period_in_deutsch_dates
         end
@@ -68,7 +68,8 @@ module PragmaticSegmenter
         ).freeze
 
         def replace
-          @text = text.apply(
+          @text = Rule.apply(
+            text,
             @language::PossessiveAbbreviationRule,
             @language::SingleLetterAbbreviationRules::All,
             SingleLowerCaseLetterRule,
@@ -76,7 +77,7 @@ module PragmaticSegmenter
 
           @text = search_for_abbreviations_in_string(@text)
           @text = replace_multi_period_abbreviations(@text)
-          @text.apply(Languages::Common::AmPmRules::All)
+          Rule.apply(@text, Languages::Common::AmPmRules::All)
           replace_abbreviation_as_sentence_boundary(@text)
         end
 

data/lib/pragmatic_segmenter/languages/japanese.rb

@@ -17,7 +17,7 @@ module PragmaticSegmenter
         private
 
         def remove_newline_in_middle_of_word
-          @text.apply NewLineInMiddleOfWordRule
+          Rule.apply @text, NewLineInMiddleOfWordRule
         end
       end
 

data/lib/pragmatic_segmenter/languages/kazakh.rb

@@ -23,7 +23,7 @@ module PragmaticSegmenter
 
         def between_punctuation(txt)
           super(txt)
-          txt.apply(QuestionMarkFollowedByDashLowercaseRule, ExclamationMarkFollowedByDashLowercaseRule)
+          Rule.apply(txt, QuestionMarkFollowedByDashLowercaseRule, ExclamationMarkFollowedByDashLowercaseRule)
         end
       end
 
@@ -35,7 +35,7 @@ module PragmaticSegmenter
 
         def replace
           super
-          @text.apply(SingleUpperCaseCyrillicLetterAtStartOfLineRule, SingleUpperCaseCyrillicLetterRule)
+          Rule.apply(@text, SingleUpperCaseCyrillicLetterAtStartOfLineRule, SingleUpperCaseCyrillicLetterRule)
         end
       end
     end

data/lib/pragmatic_segmenter/list.rb

@@ -48,7 +48,7 @@ module PragmaticSegmenter
 
     attr_reader :text
     def initialize(text:)
-      @text = Text.new(text)
+      @text = text.dup
     end
 
     def add_line_break
@@ -68,13 +68,13 @@ module PragmaticSegmenter
     def format_numbered_list_with_parens
       replace_parens_in_numbered_list
       add_line_breaks_for_numbered_list_with_parens
-      @text.apply(ListMarkerRule)
+      Rule.apply(@text, ListMarkerRule)
     end
 
     def format_numbered_list_with_periods
       replace_periods_in_numbered_list
       add_line_breaks_for_numbered_list_with_periods
-      @text.apply(SubstituteListPeriodRule)
+      Rule.apply(@text, SubstituteListPeriodRule)
     end
 
     def format_alphabetical_lists
@@ -93,7 +93,7 @@ module PragmaticSegmenter
 
     def add_line_breaks_for_numbered_list_with_periods
       if @text.include?('♨') && @text !~ /♨.+\n.+♨|♨.+\r.+♨/ && @text !~ /for\s\d{1,2}♨\s[a-z]/
-        @text.apply(SpaceBetweenListItemsFirstRule, SpaceBetweenListItemsSecondRule)
+        Rule.apply(@text, SpaceBetweenListItemsFirstRule, SpaceBetweenListItemsSecondRule)
       end
     end
 
@@ -105,7 +105,7 @@ module PragmaticSegmenter
 
     def add_line_breaks_for_numbered_list_with_parens
       if @text.include?('☝') && @text !~ /☝.+\n.+☝|☝.+\r.+☝/
-        @text.apply(SpaceBetweenListItemsThirdRule)
+        Rule.apply(@text, SpaceBetweenListItemsThirdRule)
       end
     end
 

data/lib/pragmatic_segmenter/processor.rb

@@ -24,9 +24,9 @@ module PragmaticSegmenter
       replace_numbers
       replace_continuous_punctuation
       replace_periods_before_numeric_references
-      @text.apply(@language::Abbreviations::WithMultiplePeriodsAndEmailRule)
-      @text.apply(@language::GeoLocationRule)
-      @text.apply(@language::FileFormatRule)
+      Rule.apply(@text, @language::Abbreviations::WithMultiplePeriodsAndEmailRule)
+      Rule.apply(@text, @language::GeoLocationRule)
+      Rule.apply(@text, @language::FileFormatRule)
       split_into_segments
     end
 
@@ -34,18 +34,19 @@ module PragmaticSegmenter
 
     def split_into_segments
       check_for_parens_between_quotes(@text).split("\r")
-         .map! { |segment| segment.apply(@language::SingleNewLineRule, @language::EllipsisRules::All) }
+         .map! { |segment| Rule.apply(segment, @language::SingleNewLineRule, @language::EllipsisRules::All) }
          .map { |segment| check_for_punctuation(segment) }.flatten
-         .map! { |segment| segment.apply(@language::SubSymbolsRules::All) }
+         .map! { |segment| Rule.apply(segment, @language::SubSymbolsRules::All) }
          .map { |segment| post_process_segments(segment) }
          .flatten.compact.delete_if(&:empty?)
-         .map! { |segment| segment.apply(@language::SubSingleQuoteRule) }
+         .map! { |segment| Rule.apply(segment, @language::SubSingleQuoteRule) }
     end
 
     def post_process_segments(txt)
       return txt if txt.length < 2 && txt =~ /\A[a-zA-Z]*\Z/
       return if consecutive_underscore?(txt) || txt.length < 2
-      txt.apply(
+      Rule.apply(
+        txt,
         @language::ReinsertEllipsisRules::All,
         @language::ExtraWhiteSpaceRule
       )
@@ -91,7 +92,8 @@ module PragmaticSegmenter
       txt << 'ȸ' unless @language::Punctuations.any? { |p| txt[-1].include?(p) }
       ExclamationWords.apply_rules(txt)
       between_punctuation(txt)
-      txt = txt.apply(
+      txt = Rule.apply(
+        txt,
         @language::DoublePunctuationRules::All,
         @language::QuestionMarkInQuotationRule,
         @language::ExclamationPointRules::All
@@ -101,7 +103,7 @@ module PragmaticSegmenter
     end
 
     def replace_numbers
-      @text.apply @language::Numbers::All
+      Rule.apply @text, @language::Numbers::All
     end
 
     def abbreviations_replacer
@@ -129,8 +131,8 @@ module PragmaticSegmenter
     end
 
     def sentence_boundary_punctuation(txt)
-      txt = txt.apply @language::ReplaceColonBetweenNumbersRule if defined? @language::ReplaceColonBetweenNumbersRule
-      txt = txt.apply @language::ReplaceNonSentenceBoundaryCommaRule if defined? @language::ReplaceNonSentenceBoundaryCommaRule
+      txt = Rule.apply txt, @language::ReplaceColonBetweenNumbersRule if defined? @language::ReplaceColonBetweenNumbersRule
+      txt = Rule.apply txt, @language::ReplaceNonSentenceBoundaryCommaRule if defined? @language::ReplaceNonSentenceBoundaryCommaRule
 
       txt.scan(@language::SENTENCE_BOUNDARY_REGEX)
     end

data/lib/pragmatic_segmenter/punctuation_replacer.rb

@@ -45,9 +45,9 @@ module PragmaticSegmenter
 
     def replace_punctuation(array)
       return if !array || array.empty?
-      @text.apply(Rules::EscapeRegexReservedCharacters::All)
+      Rule.apply(@text, Rules::EscapeRegexReservedCharacters::All)
       array.each do |a|
-        a.apply(Rules::EscapeRegexReservedCharacters::All)
+        Rule.apply(a, Rules::EscapeRegexReservedCharacters::All)
         sub = sub_characters(a, '.', '∯')
         sub_1 = sub_characters(sub, '。', '&ᓰ&')
         sub_2 = sub_characters(sub_1, '．', '&ᓱ&')
@@ -59,7 +59,7 @@ module PragmaticSegmenter
           sub_7 = sub_characters(sub_6, "'", '&⎋&')
         end
       end
-      @text.apply(Rules::SubEscapedRegexReservedCharacters::All)
+      Rule.apply(@text, Rules::SubEscapedRegexReservedCharacters::All)
     end
 
     def sub_characters(string, char_a, char_b)

data/lib/pragmatic_segmenter/types.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
 module PragmaticSegmenter
-  Rule = Struct.new(:pattern, :replacement)
-
-  class Text < String
-    def apply(*rules)
-      rules.flatten.each do |rule|
-        self.gsub!(rule.pattern, rule.replacement)
+  class Rule < Struct.new(:pattern, :replacement)
+    class << self
+      def apply(str, *rules)
+        rules.flatten.each do |rule|
+          str.gsub!(rule.pattern, rule.replacement)
+        end
+        str
       end
-      self
     end
   end
 end

data/lib/pragmatic_segmenter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PragmaticSegmenter
-  VERSION = "0.3.22"
+  VERSION = "0.3.24"
 end

data/lib/unicode.rb

@@ -0,0 +1,5 @@
+module Unicode
+  def self.downcase(text)
+    text.downcase
+  end
+end

data/pragmatic_segmenter.gemspec

@@ -18,9 +18,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "unicode"
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler", ">= 1.7"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "stackprof"
 end

metadata

@@ -1,57 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.3.22
+  version: 0.3.24
 platform: ruby
 authors:
 - Kevin S. Dias
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-09-23 00:00:00.000000000 Z
+date: 2024-08-11 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: unicode
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +122,7 @@ files:
 - lib/pragmatic_segmenter/segmenter.rb
 - lib/pragmatic_segmenter/types.rb
 - lib/pragmatic_segmenter/version.rb
+- lib/unicode.rb
 - pragmatic_segmenter.gemspec
 - spec/performance_spec.rb
 - spec/pragmatic_segmenter/languages/amharic_spec.rb
@@ -166,7 +153,7 @@ homepage: https://github.com/diasks2/pragmatic_segmenter
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -181,9 +168,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
-signing_key: 
+rubygems_version: 3.3.26
+signing_key:
 specification_version: 4
 summary: A rule-based sentence boundary detection gem that works out-of-the-box across
   many languages