pragmatic_segmenter 0.3.18 → 0.3.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3bb581e56e988521adc41dbb94bc7281ee7dfa95
-  data.tar.gz: 0c3b6fe877a5d39d36053b7ed68a860b5d17779b
+SHA256:
+  metadata.gz: 2c66c757c1b4bd8d090e88d7db6c627720f58f6f26e6fab9916a20e8bc15471c
+  data.tar.gz: da3a9088f72c90ddde6f0deda67d3e3b4ea3bed317970416deef794e0f594d89
 SHA512:
-  metadata.gz: 1b1dd64b5a382e8bb7ed5d79fbb9264565d71088f234ba4a9bd7cae47184e1c64f78b32a72f39326aa31936c6c6742aa2ff75cd75cd1b328987a6061a4d2534b
-  data.tar.gz: c150b178c93b7183300559e89c117cf8f9f93adf6ef33790a3ce0b292c66588fe6461c724210f7f93e078d2d08a10e995d7234bad963f8d5aa1c52c378effe5e
+  metadata.gz: 503c52965b2f98eebbc24e1215204c45307958a0279d56834e0c929d18625e81ac8c5c78779efb1a5946b5fdda5d8496b54a72b009ad6b2a597a70c4ba0fff66
+  data.tar.gz: f23773139a3a6d9f45cecaacabb363a7fb825a21eb76b40514abf4d0407191ed3b1afa887a5bc5328626abe2dbac5864895add62a1da036234036984d19a3454

data/.gitignore

@@ -12,4 +12,5 @@
 *.o
 *.a
 mkmf.log
-.DS_Store
+.DS_Store
+.vscode/launch.json

data/NEWS

@@ -1,3 +1,27 @@
+0.3.22 (2021-05-03):
+
+* Improvement: Refactor for Ruby 3.0 compatibility
+
+0.3.22 (2018-09-23):
+
+* Improvement: Initial support for Kazakh
+
+0.3.21 (2018-08-30):
+
+* Improvement: Add support for file formats
+* Improvement: Add support for numeric references at the end of a sentence (i.e. Wikipedia references)
+
+0.3.20 (2018-08-28):
+
+* Improvement: Handle slanted single quotation as a single quote
+* Bug Fix: The text contains a single character abbreviation as part of a list
+* Bug Fix: Chinese book quotes
+* Improvement: Add viz as abbreviation
+
+0.3.19 (2018-07-19):
+
+* Bug Fix: A parenthetical following an abbreviation is now included as part of the same segment. Example: "The parties to this Agreement are PragmaticSegmenterExampleCompanyA Inc. (“Company A”), and PragmaticSegmenterExampleCompanyB Inc. (“Company B”)." is now treated as one segment.
+
 0.3.18 (2018-03-27):
 
 * Improvement: Performance optimizations

data/README.md

@@ -1,6 +1,6 @@
 # Pragmatic Segmenter
 
-[![Gem Version](https://badge.fury.io/rb/pragmatic_segmenter.svg)](http://badge.fury.io/rb/pragmatic_segmenter) [![Code Climate](https://codeclimate.com/github/diasks2/pragmatic_segmenter/badges/gpa.svg)](https://codeclimate.com/github/diasks2/pragmatic_segmenter) [![Build Status](https://travis-ci.org/diasks2/pragmatic_segmenter.png)](https://travis-ci.org/diasks2/pragmatic_segmenter) [![Test Coverage](https://codeclimate.com/github/diasks2/pragmatic_segmenter/badges/coverage.svg)](https://codeclimate.com/github/diasks2/pragmatic_segmenter) [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/diasks2/pragmatic_segmenter/blob/master/LICENSE.txt)
+[![Gem Version](https://badge.fury.io/rb/pragmatic_segmenter.svg)](http://badge.fury.io/rb/pragmatic_segmenter) [![Build Status](https://travis-ci.org/diasks2/pragmatic_segmenter.png)](https://travis-ci.org/diasks2/pragmatic_segmenter) [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/diasks2/pragmatic_segmenter/blob/master/LICENSE.txt)
 
 Pragmatic Segmenter is a rule-based sentence boundary detection gem that works out-of-the-box across many languages.
 
@@ -433,6 +433,12 @@ Was sind die Konsequenzen der Abstimmung vom 12. Juni?
 => ["Was sind die Konsequenzen der Abstimmung vom 12. Juni?"]
 ```
 
+4.) **Cardinal numbers at end of sentence** *Credit: Dr. Michael Ustaszewski*
+```
+Die Information steht auf Seite 12. Dort kannst du nachlesen.
+=> ["Die Information steht auf Seite 12.", "Dort kannst du nachlesen."]
+```
+
 #### Golden Rules (Japanese)
 
 1.) **Simple period to end sentence**
@@ -865,6 +871,25 @@ To test the relative performance of different segmentation tools and libraries I
 **Version 0.3.18**
 * Performance optimizations
 
+**Version 0.3.19**
+* Treat a parenthetical following an abbreviation as part of the same segment
+
+**Version 0.3.20**
+* Handle slanted single quotation as a single quote
+* Handle a single character abbreviation as part of a list
+* Add support for Chinese caret brackets
+* Add viz as abbreviation
+
+**Version 0.3.21**
+* Add support for file formats
+* Add support for numeric references at the end of a sentence (i.e. Wikipedia references)
+
+**Version 0.3.22**
+* Add initial support and tests for Kazakh
+
+**Version 0.3.23**
+* Refactor for Ruby 3.0 compatibility
+
 ## Contributing
 
 If you find a text that is incorrectly segmented using this gem, please submit an issue.
@@ -875,6 +900,11 @@ If you find a text that is incorrectly segmented using this gem, please submit a
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new Pull Request
 
+## Ports
+
+* [C# - PragmaticSegmenterNet](https://github.com/UglyToad/PragmaticSegmenterNet)
+* [Python - pySBD](https://github.com/nipunsadvilkar/pySBD)
+
 ## License
 
 The MIT License (MIT)

data/lib/pragmatic_segmenter/abbreviation_replacer.rb

@@ -10,18 +10,19 @@ module PragmaticSegmenter
 
     attr_reader :text
     def initialize(text:, language: )
-      @text = Text.new(text)
+      @text = text.dup
       @language = language
     end
 
     def replace
-      @text.apply(@language::PossessiveAbbreviationRule,
+      Rule.apply(@text,
+        @language::PossessiveAbbreviationRule,
         @language::KommanditgesellschaftRule,
         @language::SingleLetterAbbreviationRules::All)
 
       @text = search_for_abbreviations_in_string(@text)
       @text = replace_multi_period_abbreviations(@text)
-      @text.apply(@language::AmPmRules::All)
+      Rule.apply(@text, @language::AmPmRules::All)
       replace_abbreviation_as_sentence_boundary(@text)
     end
 
@@ -108,7 +109,7 @@ module PragmaticSegmenter
     end
 
     def replace_period_of_abbr(txt, abbr)
-      txt.gsub!(/(?<=\s#{abbr.strip})\.(?=((\.|\:|-|\?)|(\s([a-z]|I\s|I'm|I'll|\d))))|(?<=^#{abbr.strip})\.(?=((\.|\:|\?)|(\s([a-z]|I\s|I'm|I'll|\d))))/, '∯')
+      txt.gsub!(/(?<=\s#{abbr.strip})\.(?=((\.|\:|-|\?)|(\s([a-z]|I\s|I'm|I'll|\d|\())))|(?<=^#{abbr.strip})\.(?=((\.|\:|\?)|(\s([a-z]|I\s|I'm|I'll|\d))))/, '∯')
       txt.gsub!(/(?<=\s#{abbr.strip})\.(?=,)|(?<=^#{abbr.strip})\.(?=,)/, '∯')
       txt
     end

data/lib/pragmatic_segmenter/between_punctuation.rb

@@ -8,6 +8,8 @@ module PragmaticSegmenter
     # Rubular: http://rubular.com/r/2YFrKWQUYi
     BETWEEN_SINGLE_QUOTES_REGEX = /(?<=\s)'(?:[^']|'[a-zA-Z])*'/
 
+    BETWEEN_SINGLE_QUOTE_SLANTED_REGEX = /(?<=\s)‘(?:[^’]|’[a-zA-Z])*’/
+
     # Rubular: http://rubular.com/r/3Pw1QlXOjd
     BETWEEN_DOUBLE_QUOTES_REGEX = /"(?>[^"\\]+|\\{2}|\\.)*"/
 
@@ -42,6 +44,7 @@ module PragmaticSegmenter
 
     def sub_punctuation_between_quotes_and_parens(txt)
       sub_punctuation_between_single_quotes(txt)
+      sub_punctuation_between_single_quote_slanted(txt)
       sub_punctuation_between_double_quotes(txt)
       sub_punctuation_between_square_brackets(txt)
       sub_punctuation_between_parens(txt)
@@ -74,6 +77,13 @@ module PragmaticSegmenter
       end
     end
 
+    def sub_punctuation_between_single_quote_slanted(txt)
+      PragmaticSegmenter::PunctuationReplacer.new(
+        matches_array: txt.scan(BETWEEN_SINGLE_QUOTE_SLANTED_REGEX),
+        text: txt
+      ).replace
+    end
+
     def sub_punctuation_between_double_quotes(txt)
       PragmaticSegmenter::PunctuationReplacer.new(
         matches_array: btwn_dbl_quote(txt),

data/lib/pragmatic_segmenter/cleaner.rb

@@ -11,7 +11,7 @@ module PragmaticSegmenter
 
     attr_reader :text, :doc_type
     def initialize(text:, doc_type: nil, language: Languages::Common)
-      @text = Text.new(text)
+      @text = text.dup
       @doc_type = doc_type
       @language = language
     end
@@ -37,10 +37,10 @@ module PragmaticSegmenter
       replace_newlines
       replace_escaped_newlines
 
-      @text.apply(HTML::All)
+      Rule.apply(@text, HTML::All)
 
       replace_punctuation_in_brackets
-      @text.apply(InlineFormattingRule)
+      Rule.apply(@text, InlineFormattingRule)
       clean_quotations
       clean_table_of_contents
       check_for_no_space_in_between_sentences
@@ -72,7 +72,7 @@ module PragmaticSegmenter
       if word =~ regex
         unless URL_EMAIL_KEYWORDS.any? { |web| word =~ /#{web}/ }
           unless abbreviations.any? { |abbr| word =~ /#{abbr}/i }
-            new_word = word.dup.apply(rule)
+            new_word = Rule.apply(word.dup, rule)
             txt.gsub!(/#{Regexp.escape(word)}/, new_word)
           end
         end
@@ -92,45 +92,45 @@ module PragmaticSegmenter
     end
 
     def remove_newline_in_middle_of_word
-      @text.apply NewLineInMiddleOfWordRule
+      Rule.apply @text, NewLineInMiddleOfWordRule
     end
 
     def replace_escaped_newlines
-      @text.apply EscapedNewLineRule, EscapedCarriageReturnRule,
+      Rule.apply @text, EscapedNewLineRule, EscapedCarriageReturnRule,
         TypoEscapedNewLineRule, TypoEscapedCarriageReturnRule
     end
 
     def replace_double_newlines
-      @text.apply DoubleNewLineWithSpaceRule, DoubleNewLineRule
+      Rule.apply @text, DoubleNewLineWithSpaceRule, DoubleNewLineRule
     end
 
     def replace_newlines
       if doc_type.eql?('pdf')
         remove_pdf_line_breaks
       else
-        @text.apply NewLineFollowedByPeriodRule,
+        Rule.apply @text, NewLineFollowedByPeriodRule,
           ReplaceNewlineWithCarriageReturnRule
       end
     end
 
     def remove_pdf_line_breaks
-      @text.apply NewLineFollowedByBulletRule,
+      Rule.apply @text, NewLineFollowedByBulletRule,
 
         PDF::NewLineInMiddleOfSentenceRule,
         PDF::NewLineInMiddleOfSentenceNoSpacesRule
     end
 
     def clean_quotations
-      @text.apply QuotationsFirstRule, QuotationsSecondRule
+      Rule.apply @text, QuotationsFirstRule, QuotationsSecondRule
     end
 
     def clean_table_of_contents
-      @text.apply TableOfContentsRule, ConsecutivePeriodsRule,
+      Rule.apply @text, TableOfContentsRule, ConsecutivePeriodsRule,
         ConsecutiveForwardSlashRule
     end
 
     def clean_consecutive_characters
-      @text.apply ConsecutivePeriodsRule, ConsecutiveForwardSlashRule
+      Rule.apply @text, ConsecutivePeriodsRule, ConsecutiveForwardSlashRule
     end
   end
 end

data/lib/pragmatic_segmenter/languages.rb

@@ -26,6 +26,7 @@ require 'pragmatic_segmenter/languages/polish'
 require 'pragmatic_segmenter/languages/chinese'
 require 'pragmatic_segmenter/languages/bulgarian'
 require 'pragmatic_segmenter/languages/danish'
+require 'pragmatic_segmenter/languages/kazakh'
 
 module PragmaticSegmenter
   module Languages
@@ -49,7 +50,8 @@ module PragmaticSegmenter
       'nl' => Dutch,
       'pl' => Polish,
       'zh' => Chinese,
-      'da' => Danish
+      'da' => Danish,
+      'kk' => Kazakh
     }
 
     def self.get_language_by_code(code)

data/lib/pragmatic_segmenter/languages/chinese.rb

@@ -8,6 +8,32 @@ module PragmaticSegmenter
       class AbbreviationReplacer < AbbreviationReplacer
         SENTENCE_STARTERS = [].freeze
       end
+
+      class BetweenPunctuation < PragmaticSegmenter::BetweenPunctuation
+        BETWEEN_DOUBLE_ANGLE_QUOTATION_MARK_REGEX = /《(?>[^》\\]+|\\{2}|\\.)*》/
+        BETWEEN_L_BRACKET_REGEX = /「(?>[^」\\]+|\\{2}|\\.)*」/
+        private
+
+        def sub_punctuation_between_quotes_and_parens(txt)
+          super
+          sub_punctuation_between_double_angled_quotation_marks(txt)
+          sub_punctuation_between_l_bracket(txt)
+        end
+        
+        def sub_punctuation_between_double_angled_quotation_marks(txt)
+          PunctuationReplacer.new(
+            matches_array: txt.scan(BETWEEN_DOUBLE_ANGLE_QUOTATION_MARK_REGEX),
+            text: txt
+          ).replace
+        end
+        
+        def sub_punctuation_between_l_bracket(txt)
+          PunctuationReplacer.new(
+            matches_array: txt.scan(BETWEEN_L_BRACKET_REGEX),
+            text: txt
+          ).replace
+        end
+      end
     end
   end
 end

data/lib/pragmatic_segmenter/languages/common.rb

@@ -11,7 +11,7 @@ module PragmaticSegmenter
 
       # Defines the abbreviations for each language (if available)
       module Abbreviation
-        ABBREVIATIONS = Set.new(['adj', 'adm', 'adv', 'al', 'ala', 'alta', 'apr', 'arc', 'ariz', 'ark', 'art', 'assn', 'asst', 'attys', 'aug', 'ave', 'bart', 'bld', 'bldg', 'blvd', 'brig', 'bros', 'btw', 'cal', 'calif', 'capt', 'cl', 'cmdr', 'co', 'col', 'colo', 'comdr', 'con', 'conn', 'corp', 'cpl', 'cres', 'ct', 'd.phil', 'dak', 'dec', 'del', 'dept', 'det', 'dist', 'dr', 'dr.phil', 'dr.philos', 'drs', 'e.g', 'ens', 'esp', 'esq', 'etc', 'exp', 'expy', 'ext', 'feb', 'fed', 'fla', 'ft', 'fwy', 'fy', 'ga', 'gen', 'gov', 'hon', 'hosp', 'hr', 'hway', 'hwy', 'i.e', 'ia', 'id', 'ida', 'ill', 'inc', 'ind', 'ing', 'insp', 'is', 'jan', 'jr', 'jul', 'jun', 'kan', 'kans', 'ken', 'ky', 'la', 'lt', 'ltd', 'maj', 'man', 'mar', 'mass', 'may', 'md', 'me', 'med', 'messrs', 'mex', 'mfg', 'mich', 'min', 'minn', 'miss', 'mlle', 'mm', 'mme', 'mo', 'mont', 'mr', 'mrs', 'ms', 'msgr', 'mssrs', 'mt', 'mtn', 'neb', 'nebr', 'nev', 'no', 'nos', 'nov', 'nr', 'oct', 'ok', 'okla', 'ont', 'op', 'ord', 'ore', 'p', 'pa', 'pd', 'pde', 'penn', 'penna', 'pfc', 'ph', 'ph.d', 'pl', 'plz', 'pp', 'prof', 'pvt', 'que', 'rd', 'rs', 'ref', 'rep', 'reps', 'res', 'rev', 'rt', 'sask', 'sec', 'sen', 'sens', 'sep', 'sept', 'sfc', 'sgt', 'sr', 'st', 'supt', 'surg', 'tce', 'tenn', 'tex', 'univ', 'usafa', 'u.s', 'ut', 'va', 'v', 'ver', 'vs', 'vt', 'wash', 'wis', 'wisc', 'wy', 'wyo', 'yuk']).freeze
+        ABBREVIATIONS = Set.new(['adj', 'adm', 'adv', 'al', 'ala', 'alta', 'apr', 'arc', 'ariz', 'ark', 'art', 'assn', 'asst', 'attys', 'aug', 'ave', 'bart', 'bld', 'bldg', 'blvd', 'brig', 'bros', 'btw', 'cal', 'calif', 'capt', 'cl', 'cmdr', 'co', 'col', 'colo', 'comdr', 'con', 'conn', 'corp', 'cpl', 'cres', 'ct', 'd.phil', 'dak', 'dec', 'del', 'dept', 'det', 'dist', 'dr', 'dr.phil', 'dr.philos', 'drs', 'e.g', 'ens', 'esp', 'esq', 'etc', 'exp', 'expy', 'ext', 'feb', 'fed', 'fla', 'ft', 'fwy', 'fy', 'ga', 'gen', 'gov', 'hon', 'hosp', 'hr', 'hway', 'hwy', 'i.e', 'ia', 'id', 'ida', 'ill', 'inc', 'ind', 'ing', 'insp', 'is', 'jan', 'jr', 'jul', 'jun', 'kan', 'kans', 'ken', 'ky', 'la', 'lt', 'ltd', 'maj', 'man', 'mar', 'mass', 'may', 'md', 'me', 'med', 'messrs', 'mex', 'mfg', 'mich', 'min', 'minn', 'miss', 'mlle', 'mm', 'mme', 'mo', 'mont', 'mr', 'mrs', 'ms', 'msgr', 'mssrs', 'mt', 'mtn', 'neb', 'nebr', 'nev', 'no', 'nos', 'nov', 'nr', 'oct', 'ok', 'okla', 'ont', 'op', 'ord', 'ore', 'p', 'pa', 'pd', 'pde', 'penn', 'penna', 'pfc', 'ph', 'ph.d', 'pl', 'plz', 'pp', 'prof', 'pvt', 'que', 'rd', 'rs', 'ref', 'rep', 'reps', 'res', 'rev', 'rt', 'sask', 'sec', 'sen', 'sens', 'sep', 'sept', 'sfc', 'sgt', 'sr', 'st', 'supt', 'surg', 'tce', 'tenn', 'tex', 'univ', 'usafa', 'u.s', 'ut', 'va', 'v', 'ver', 'viz', 'vs', 'vt', 'wash', 'wis', 'wisc', 'wy', 'wyo', 'yuk']).freeze
         PREPOSITIVE_ABBREVIATIONS = Set.new(['adm', 'attys', 'brig', 'capt', 'cmdr', 'col', 'cpl', 'det', 'dr', 'gen', 'gov', 'ing', 'lt', 'maj', 'mr', 'mrs', 'ms', 'mt', 'messrs', 'mssrs', 'prof', 'ph', 'rep', 'reps', 'rev', 'sen', 'sens', 'sgt', 'st', 'supt', 'v', 'vs']).freeze
         NUMBER_ABBREVIATIONS = Set.new(['art', 'ext', 'no', 'nos', 'p', 'pp']).freeze
       end
@@ -24,6 +24,8 @@ module PragmaticSegmenter
       # Rubular: http://rubular.com/r/G2opjedIm9
       GeoLocationRule = Rule.new(/(?<=[a-zA-z]°)\.(?=\s*\d+)/, '∯')
 
+      FileFormatRule = Rule.new(/(?<=\s)\.(?=(jpe?g|png|gif|tiff?|pdf|ps|docx?|xlsx?|svg|bmp|tga|exif|odt|html?|txt|rtf|bat|sxw|xml|zip|exe|msi|blend|wmv|mp[34]|pptx?|flac|rb|cpp|cs|js)\s)/, '∯')
+
       SingleNewLineRule = Rule.new(/\n/, 'ȹ')
 
       module DoublePunctuationRules

data/lib/pragmatic_segmenter/languages/common/numbers.rb

@@ -47,6 +47,8 @@ module PragmaticSegmenter
       # Rubular: http://rubular.com/r/mQ8Es9bxtk
       CONTINUOUS_PUNCTUATION_REGEX = /(?<=\S)(!|\?){3,}(?=(\s|\z|$))/
 
+      NUMBERED_REFERENCE_REGEX = /(?<=[^\d\s])(\.|∯)((\[(\d{1,3},?\s?-?\s?)*\b\d{1,3}\])+|((\d{1,3}\s?)*\d{1,3}))(\s)(?=[A-Z])/
+
       # Rubular: http://rubular.com/r/yqa4Rit8EY
       PossessiveAbbreviationRule = Rule.new(/\.(?='s\s)|\.(?='s$)|\.(?='s\z)/, '∯')
 
@@ -76,10 +78,10 @@ module PragmaticSegmenter
       # replaces the periods.
       module SingleLetterAbbreviationRules
         # Rubular: http://rubular.com/r/e3H6kwnr6H
-        SingleUpperCaseLetterAtStartOfLineRule = Rule.new(/(?<=^[A-Z])\.(?=\s)/, '∯')
+        SingleUpperCaseLetterAtStartOfLineRule = Rule.new(/(?<=^[A-Z])\.(?=,?\s)/, '∯')
 
         # Rubular: http://rubular.com/r/gitvf0YWH4
-        SingleUpperCaseLetterRule = Rule.new(/(?<=\s[A-Z])\.(?=\s)/, '∯')
+        SingleUpperCaseLetterRule = Rule.new(/(?<=\s[A-Z])\.(?=,?\s)/, '∯')
 
         All = [
           SingleUpperCaseLetterAtStartOfLineRule,

data/lib/pragmatic_segmenter/languages/deutsch.rb

@@ -47,7 +47,7 @@ module PragmaticSegmenter
         private
 
         def replace_numbers
-          @text.apply Numbers::All
+          Rule.apply @text, Numbers::All
 
           replace_period_in_deutsch_dates
         end
@@ -68,7 +68,8 @@ module PragmaticSegmenter
         ).freeze
 
         def replace
-          @text = text.apply(
+          @text = Rule.apply(
+            text,
             @language::PossessiveAbbreviationRule,
             @language::SingleLetterAbbreviationRules::All,
             SingleLowerCaseLetterRule,
@@ -76,7 +77,7 @@ module PragmaticSegmenter
 
           @text = search_for_abbreviations_in_string(@text)
           @text = replace_multi_period_abbreviations(@text)
-          @text.apply(Languages::Common::AmPmRules::All)
+          Rule.apply(@text, Languages::Common::AmPmRules::All)
           replace_abbreviation_as_sentence_boundary(@text)
         end
 

data/lib/pragmatic_segmenter/languages/japanese.rb

@@ -17,7 +17,7 @@ module PragmaticSegmenter
         private
 
         def remove_newline_in_middle_of_word
-          @text.apply NewLineInMiddleOfWordRule
+          Rule.apply @text, NewLineInMiddleOfWordRule
         end
       end
 

data/lib/pragmatic_segmenter/languages/kazakh.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module PragmaticSegmenter
+  module Languages
+    module Kazakh
+      include Languages::Common
+
+      MULTI_PERIOD_ABBREVIATION_REGEX = /\b\p{Cyrillic}(?:\.\s?\p{Cyrillic})+[.]|b[a-z](?:\.[a-z])+[.]/i
+
+      module Abbreviation
+        ABBREVIATIONS = Set.new(['afp', 'anp', 'atp', 'bae', 'bg', 'bp', 'cam', 'cctv', 'cd', 'cez', 'cgi', 'cnpc', 'farc', 'fbi', 'eiti', 'epo', 'er', 'gp', 'gps', 'has', 'hiv', 'hrh', 'http', 'icu', 'idf', 'imd', 'ime', 'icu', 'idf', 'ip', 'iso', 'kaz', 'kpo', 'kpa', 'kz', 'kz', 'mri', 'nasa', 'nba', 'nbc', 'nds', 'ohl', 'omlt', 'ppm', 'pda', 'pkk', 'psm', 'psp', 'raf', 'rss', 'rtl', 'sas', 'sme', 'sms', 'tnt', 'udf', 'uefa', 'usb', 'utc', 'x', 'zdf', 'әқбк', 'әқбк', 'аақ', 'авг.', 'aбб', 'аек', 'ак', 'ақ', 'акцион.', 'акср', 'ақш', 'англ', 'аөсшк', 'апр', 'м.', 'а.', 'р.', 'ғ.', 'апр.', 'аум.', 'ацат', 'әч', 'т. б.', 'б. з. б.', 'б. з. б.', 'б. з. д.', 'б. з. д.', 'биікт.', 'б. т.', 'биол.', 'биохим', 'бө', 'б. э. д.', 'бта', 'бұұ', 'вич', 'всоонл', 'геогр.', 'геол.', 'гленкор', 'гэс', 'қк', 'км', 'г', 'млн', 'млрд', 'т', 'ғ. с.', 'ғ.', 'қ.', 'ғ.', 'дек.', 'днқ', 'дсұ', 'еақк', 'еқыұ', 'ембімұнайгаз', 'ео', 'еуразэқ', 'еуроодақ', 'еұу', 'ж.', 'ж.', 'жж.', 'жоо', 'жіө', 'жсдп', 'жшс', 'іім', 'инта', 'исаф', 'камаз', 'кгб', 'кеу', 'кг', 'км²', 'км²', 'км³', 'км³', 'кимеп', 'кср', 'ксро', 'кокп', 'кхдр', 'қазатомпром', 'қазкср', 'қазұу', 'қазмұнайгаз', 'қазпошта', 'қазтаг', 'қазұу', 'қкп', 'қмдб', 'қр', 'қхр', 'лат.', 'м²', 'м²', 'м³', 'м³', 'магатэ', 'май.', 'максам', 'мб', 'мвт', 'мемл', 'м', 'мсоп', 'мтк', 'мыс.', 'наса', 'нато', 'нквд', 'нояб.', 'обл.', 'огпу', 'окт.', 'оңт.', 'опек', 'оеб', 'өзенмұнайгаз', 'өф', 'пәк', 'пед.', 'ркфср', 'рнқ', 'рсфср', 'рф', 'свс', 'сву', 'сду', 'сес', 'сент.', 'см', 'снпс', 'солт.', 'солт.', 'сооно', 'ссро', 'сср', 'ссср', 'ссс', 'сэс', 'дк', 'т. б.', 'т', 'тв', 'тереңд.', 'тех.', 'тжқ', 'тмд', 'төм.', 'трлн', 'тр', 'т.', 'и.', 'м.', 'с.', 'ш.', 'т.', 'т. с. с.', 'тэц', 'уаз', 'уефа', 'еқыұ', 'ұқк', 'ұқшұ', 'февр.', 'фққ', 'фсб', 'хим.', 'хқко', 'шұар', 'шыұ', 'экон.', 'экспо', 'цтп', 'цас', 'янв.', 'dvd', 'жкт', 'ққс', 'км', 'ацат', 'юнеско', 'ббс', 'mgm', 'жск', 'зоо', 'бсн', 'өұқ', 'оар', 'боак', 'эөкк', 'хтқо', 'әөк', 'жэк', 'хдо', 'спбму', 'аф', 'сбд', 'амт', 'гсдп', 'гсбп', 'эыдұ', 'нұсжп', 'шыұ', 'жтсх', 'хдп', 'эқк', 'фкққ', 'пиқ', 'өгк', 'мбф', 'маж', 'кота', 'тж', 'ук', 'обб', 'сбл', 'жхл', 'кмс', 'бмтрк', 'жққ', 'бхооо', 'мқо', 'ржмб', 'гулаг', 'жко', 'еэы', 'еаэы', 'кхдр', 'рфкп', 'рлдп', 'хвқ', 'мр', 'мт', 'кту', 'ртж', 'тим', 'мемдум', 'ксро', 'т.с.с', 'с.ш.', 'ш.б.', 'б.б.', 'руб', 'мин', 'акад.', 'ғ.', 'мм', 'мм.']).freeze
+        PREPOSITIVE_ABBREVIATIONS = [].freeze
+        NUMBER_ABBREVIATIONS = [].freeze
+      end
+
+      class Processor < PragmaticSegmenter::Processor
+        private
+
+        # Rubular: http://rubular.com/r/WRWy56Z5zp
+        QuestionMarkFollowedByDashLowercaseRule = Rule.new(/(?<=\p{Ll})\?(?=\s*[-—]\s*\p{Ll})/, '&ᓷ&')
+        # Rubular: http://rubular.com/r/lixxP7puSa
+        ExclamationMarkFollowedByDashLowercaseRule = Rule.new(/(?<=\p{Ll})!(?=\s*[-—]\s*\p{Ll})/, '&ᓴ&')
+
+        def between_punctuation(txt)
+          super(txt)
+          Rule.apply(txt, QuestionMarkFollowedByDashLowercaseRule, ExclamationMarkFollowedByDashLowercaseRule)
+        end
+      end
+
+      class AbbreviationReplacer < AbbreviationReplacer
+        SENTENCE_STARTERS = [].freeze
+
+        SingleUpperCaseCyrillicLetterAtStartOfLineRule = Rule.new(/(?<=^[А-ЯЁ])\.(?=\s)/, '∯')
+        SingleUpperCaseCyrillicLetterRule = Rule.new(/(?<=\s[А-ЯЁ])\.(?=\s)/, '∯')
+
+        def replace
+          super
+          Rule.apply(@text, SingleUpperCaseCyrillicLetterAtStartOfLineRule, SingleUpperCaseCyrillicLetterRule)
+        end
+      end
+    end
+  end
+end
+

data/lib/pragmatic_segmenter/list.rb

@@ -48,7 +48,7 @@ module PragmaticSegmenter
 
     attr_reader :text
     def initialize(text:)
-      @text = Text.new(text)
+      @text = text.dup
     end
 
     def add_line_break
@@ -68,13 +68,13 @@ module PragmaticSegmenter
     def format_numbered_list_with_parens
       replace_parens_in_numbered_list
       add_line_breaks_for_numbered_list_with_parens
-      @text.apply(ListMarkerRule)
+      Rule.apply(@text, ListMarkerRule)
     end
 
     def format_numbered_list_with_periods
       replace_periods_in_numbered_list
       add_line_breaks_for_numbered_list_with_periods
-      @text.apply(SubstituteListPeriodRule)
+      Rule.apply(@text, SubstituteListPeriodRule)
     end
 
     def format_alphabetical_lists
@@ -93,7 +93,7 @@ module PragmaticSegmenter
 
     def add_line_breaks_for_numbered_list_with_periods
       if @text.include?('♨') && @text !~ /♨.+\n.+♨|♨.+\r.+♨/ && @text !~ /for\s\d{1,2}♨\s[a-z]/
-        @text.apply(SpaceBetweenListItemsFirstRule, SpaceBetweenListItemsSecondRule)
+        Rule.apply(@text, SpaceBetweenListItemsFirstRule, SpaceBetweenListItemsSecondRule)
       end
     end
 
@@ -105,7 +105,7 @@ module PragmaticSegmenter
 
     def add_line_breaks_for_numbered_list_with_parens
       if @text.include?('☝') && @text !~ /☝.+\n.+☝|☝.+\r.+☝/
-        @text.apply(SpaceBetweenListItemsThirdRule)
+        Rule.apply(@text, SpaceBetweenListItemsThirdRule)
       end
     end
 

data/lib/pragmatic_segmenter/processor.rb

@@ -23,8 +23,10 @@ module PragmaticSegmenter
       replace_abbreviations
       replace_numbers
       replace_continuous_punctuation
-      @text.apply(@language::Abbreviations::WithMultiplePeriodsAndEmailRule)
-      @text.apply(@language::GeoLocationRule)
+      replace_periods_before_numeric_references
+      Rule.apply(@text, @language::Abbreviations::WithMultiplePeriodsAndEmailRule)
+      Rule.apply(@text, @language::GeoLocationRule)
+      Rule.apply(@text, @language::FileFormatRule)
       split_into_segments
     end
 
@@ -32,18 +34,19 @@ module PragmaticSegmenter
 
     def split_into_segments
       check_for_parens_between_quotes(@text).split("\r")
-         .map! { |segment| segment.apply(@language::SingleNewLineRule, @language::EllipsisRules::All) }
+         .map! { |segment| Rule.apply(segment, @language::SingleNewLineRule, @language::EllipsisRules::All) }
          .map { |segment| check_for_punctuation(segment) }.flatten
-         .map! { |segment| segment.apply(@language::SubSymbolsRules::All) }
+         .map! { |segment| Rule.apply(segment, @language::SubSymbolsRules::All) }
          .map { |segment| post_process_segments(segment) }
          .flatten.compact.delete_if(&:empty?)
-         .map! { |segment| segment.apply(@language::SubSingleQuoteRule) }
+         .map! { |segment| Rule.apply(segment, @language::SubSingleQuoteRule) }
     end
 
     def post_process_segments(txt)
       return txt if txt.length < 2 && txt =~ /\A[a-zA-Z]*\Z/
       return if consecutive_underscore?(txt) || txt.length < 2
-      txt.apply(
+      Rule.apply(
+        txt,
         @language::ReinsertEllipsisRules::All,
         @language::ExtraWhiteSpaceRule
       )
@@ -68,6 +71,10 @@ module PragmaticSegmenter
       end
     end
 
+    def replace_periods_before_numeric_references
+      @text.gsub!(@language::NUMBERED_REFERENCE_REGEX, "∯\\2\r\\7")
+    end
+
     def consecutive_underscore?(txt)
       # Rubular: http://rubular.com/r/fTF2Ff3WBL
       txt.gsub(/_{3,}/, '').length.eql?(0)
@@ -85,7 +92,8 @@ module PragmaticSegmenter
       txt << 'ȸ' unless @language::Punctuations.any? { |p| txt[-1].include?(p) }
       ExclamationWords.apply_rules(txt)
       between_punctuation(txt)
-      txt = txt.apply(
+      txt = Rule.apply(
+        txt,
         @language::DoublePunctuationRules::All,
         @language::QuestionMarkInQuotationRule,
         @language::ExclamationPointRules::All
@@ -95,7 +103,7 @@ module PragmaticSegmenter
     end
 
     def replace_numbers
-      @text.apply @language::Numbers::All
+      Rule.apply @text, @language::Numbers::All
     end
 
     def abbreviations_replacer
@@ -123,8 +131,8 @@ module PragmaticSegmenter
     end
 
     def sentence_boundary_punctuation(txt)
-      txt = txt.apply @language::ReplaceColonBetweenNumbersRule if defined? @language::ReplaceColonBetweenNumbersRule
-      txt = txt.apply @language::ReplaceNonSentenceBoundaryCommaRule if defined? @language::ReplaceNonSentenceBoundaryCommaRule
+      txt = Rule.apply txt, @language::ReplaceColonBetweenNumbersRule if defined? @language::ReplaceColonBetweenNumbersRule
+      txt = Rule.apply txt, @language::ReplaceNonSentenceBoundaryCommaRule if defined? @language::ReplaceNonSentenceBoundaryCommaRule
 
       txt.scan(@language::SENTENCE_BOUNDARY_REGEX)
     end

data/lib/pragmatic_segmenter/punctuation_replacer.rb

@@ -45,9 +45,9 @@ module PragmaticSegmenter
 
     def replace_punctuation(array)
       return if !array || array.empty?
-      @text.apply(Rules::EscapeRegexReservedCharacters::All)
+      Rule.apply(@text, Rules::EscapeRegexReservedCharacters::All)
       array.each do |a|
-        a.apply(Rules::EscapeRegexReservedCharacters::All)
+        Rule.apply(a, Rules::EscapeRegexReservedCharacters::All)
         sub = sub_characters(a, '.', '∯')
         sub_1 = sub_characters(sub, '。', '&ᓰ&')
         sub_2 = sub_characters(sub_1, '．', '&ᓱ&')
@@ -59,7 +59,7 @@ module PragmaticSegmenter
           sub_7 = sub_characters(sub_6, "'", '&⎋&')
         end
       end
-      @text.apply(Rules::SubEscapedRegexReservedCharacters::All)
+      Rule.apply(@text, Rules::SubEscapedRegexReservedCharacters::All)
     end
 
     def sub_characters(string, char_a, char_b)

data/lib/pragmatic_segmenter/types.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
 module PragmaticSegmenter
-  Rule = Struct.new(:pattern, :replacement)
-
-  class Text < String
-    def apply(*rules)
-      rules.flatten.each do |rule|
-        self.gsub!(rule.pattern, rule.replacement)
+  class Rule < Struct.new(:pattern, :replacement)
+    class << self
+      def apply(str, *rules)
+        rules.flatten.each do |rule|
+          str.gsub!(rule.pattern, rule.replacement)
+        end
+        str
       end
-      self
     end
   end
 end

data/lib/pragmatic_segmenter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PragmaticSegmenter
-  VERSION = "0.3.18"
+  VERSION = "0.3.23"
 end

data/pragmatic_segmenter.gemspec

@@ -19,8 +19,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "unicode"
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler", ">= 1.7"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "stackprof"
 end

data/spec/pragmatic_segmenter/languages/chinese_spec.rb

@@ -7,5 +7,10 @@ RSpec.describe PragmaticSegmenter::Languages::Chinese, '(zh)' do
       ps = PragmaticSegmenter::Segmenter.new(text: "安永已聯繫周怡安親屬，協助辦理簽證相關事宜，周怡安家屬1月1日晚間搭乘東方航空班機抵達上海，他們步入入境大廳時神情落寞、不發一語。周怡安來自台中，去年剛從元智大學畢業，同年9月加入安永。", language: 'zh')
       expect(ps.segment).to eq(["安永已聯繫周怡安親屬，協助辦理簽證相關事宜，周怡安家屬1月1日晚間搭乘東方航空班機抵達上海，他們步入入境大廳時神情落寞、不發一語。", "周怡安來自台中，去年剛從元智大學畢業，同年9月加入安永。"])
     end
+
+    it 'correctly segments text #002' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "我们明天一起去看《摔跤吧！爸爸》好吗？好！", language: 'zh')
+      expect(ps.segment).to eq(["我们明天一起去看《摔跤吧！爸爸》好吗？", "好！"])
+    end
   end
 end

data/spec/pragmatic_segmenter/languages/danish_spec.rb

@@ -507,7 +507,7 @@ RSpec.describe PragmaticSegmenter::Languages::Danish, "(da)" do
 
     it 'correctly segments text #048' do
       ps = PragmaticSegmenter::Segmenter.new(text: "CELLULAR COMMUNICATIONS INC. sold 1,550,000 common shares at $21.75 each yesterday, according to lead underwriter L.F. Rothschild & Co. (cited from WSJ 05/29/1987)", language: 'en')
-      expect(ps.segment).to eq(["CELLULAR COMMUNICATIONS INC. sold 1,550,000 common shares at $21.75 each yesterday, according to lead underwriter L.F. Rothschild & Co.", "(cited from WSJ 05/29/1987)"])
+      expect(ps.segment).to eq(["CELLULAR COMMUNICATIONS INC. sold 1,550,000 common shares at $21.75 each yesterday, according to lead underwriter L.F. Rothschild & Co. (cited from WSJ 05/29/1987)"])
     end
 
     it 'correctly segments text #049' do

data/spec/pragmatic_segmenter/languages/deutsch_spec.rb

@@ -17,6 +17,24 @@ RSpec.describe PragmaticSegmenter::Languages::Deutsch, "(de)" do
       ps = PragmaticSegmenter::Segmenter.new(text: "Was sind die Konsequenzen der Abstimmung vom 12. Juni?", language: "de")
       expect(ps.segment).to eq(["Was sind die Konsequenzen der Abstimmung vom 12. Juni?"])
     end
+
+    it "Numbers #004" do
+      # Credit: Dr. Michael Ustaszewski
+      # A numeral followed by a dot within the sentence should not be treated as a sentence,
+      # because the meaning of numeral + dot is that of an ordinal number.
+      # However, if the numeral + dot is in the last position of the sentence, then it is not an ordinal,
+      # but a cardinal number and thus a sentence break should be made. See the following example:
+
+      # <INPUT>Die Information steht auf Seite 12. Dort kannst du nachlesen.</INPUT>
+      # <SHOULDBE>["Die Information steht auf Seite 12.", "Dort kannst du nachlesen."]</SHOULDBE>
+      # The sentence translates as "The information can be found on page 12. You can read it there".
+
+      # That's a tricky one I guess, because the capitalisation of the word following the dot is not necessarily a clue,
+      # since German nouns are usually always capitalised.
+      skip "NOT IMPLEMENTED"
+      ps = PragmaticSegmenter::Segmenter.new(text: "Die Information steht auf Seite 12. Dort kannst du nachlesen.", language: "de")
+      expect(ps.segment).to eq(["Die Information steht auf Seite 12.", "Dort kannst du nachlesen."])
+    end
   end
 
   # Thanks to Silvia Busse for the German test examples.

data/spec/pragmatic_segmenter/languages/english_spec.rb

@@ -512,7 +512,7 @@ RSpec.describe PragmaticSegmenter::Languages::English, "(en)" do
 
     it 'correctly segments text #048' do
       ps = PragmaticSegmenter::Segmenter.new(text: "CELLULAR COMMUNICATIONS INC. sold 1,550,000 common shares at $21.75 each yesterday, according to lead underwriter L.F. Rothschild & Co. (cited from WSJ 05/29/1987)", language: 'en')
-      expect(ps.segment).to eq(["CELLULAR COMMUNICATIONS INC. sold 1,550,000 common shares at $21.75 each yesterday, according to lead underwriter L.F. Rothschild & Co.", "(cited from WSJ 05/29/1987)"])
+      expect(ps.segment).to eq(["CELLULAR COMMUNICATIONS INC. sold 1,550,000 common shares at $21.75 each yesterday, according to lead underwriter L.F. Rothschild & Co. (cited from WSJ 05/29/1987)"])
     end
 
     it 'correctly segments text #049' do
@@ -1406,5 +1406,61 @@ RSpec.describe PragmaticSegmenter::Languages::English, "(en)" do
       expect(ps.segment).to eq(["In placebo-controlled studies of all uses of Tracleer, marked decreases in hemoglobin (>15% decrease from baseline resulting in values <11 g/ dL) were observed in 6% of Tracleer-treated patients and 3% of placebo-treated patients.", "Bosentan is highly bound (>98%) to plasma proteins, mainly albumin."])
     end
 
+    it 'correctly segments text #118' do
+      text = "The parties to this Agreement are PragmaticSegmenterExampleCompanyA Inc. (“Company A”), and PragmaticSegmenterExampleCompanyB Inc. (“Company B”)."
+      ps = PragmaticSegmenter::Segmenter.new(text: text, clean: false)
+      expect(ps.segment).to eq(["The parties to this Agreement are PragmaticSegmenterExampleCompanyA Inc. (“Company A”), and PragmaticSegmenterExampleCompanyB Inc. (“Company B”)."])
+    end
+
+    it 'correctly segments text #119' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Unlike the abbreviations i.e. and e.g., viz. is used to indicate a detailed description of something stated before.")
+      expect(ps.segment).to eq(["Unlike the abbreviations i.e. and e.g., viz. is used to indicate a detailed description of something stated before."])
+    end
+
+    it 'correctly segments text #120' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "For example, ‘dragonswort… is said that it should be grown in dragon’s blood. It grows at the tops of mountains where there are groves of trees, chiefly in holy places and in the country that is called Apulia’ (translated by Anne Van Arsdall, in Medieval Herbal Remedies: The Old English Herbarium and Anglo-Saxon Medicine p. 154). The Herbal also includes lore about other plants, such as the mandrake.")
+      expect(ps.segment).to eq(["For example, ‘dragonswort… is said that it should be grown in dragon’s blood. It grows at the tops of mountains where there are groves of trees, chiefly in holy places and in the country that is called Apulia’ (translated by Anne Van Arsdall, in Medieval Herbal Remedies: The Old English Herbarium and Anglo-Saxon Medicine p. 154).", "The Herbal also includes lore about other plants, such as the mandrake."])
+    end
+
+    it 'correctly segments text #121' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Here’s the - ahem - official citation: Baker, C., Anderson, Kenneth, Martin, James, & Palen, Leysia. Modeling Open Source Software Communities, ProQuest Dissertations and Theses.")
+      expect(ps.segment).to eq(["Here’s the - ahem - official citation: Baker, C., Anderson, Kenneth, Martin, James, & Palen, Leysia.", "Modeling Open Source Software Communities, ProQuest Dissertations and Theses."])
+    end
+
+    it 'correctly segments text #122' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "These include images of various modes of transport and members of the team, all available in .jpeg format. Images can be downloaded from our website. We also offer archives as .zip files.")
+      expect(ps.segment).to eq(["These include images of various modes of transport and members of the team, all available in .jpeg format.", "Images can be downloaded from our website.", "We also offer archives as .zip files."])
+    end
+
+    it 'correctly segments text #123' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Saint Maximus (died 250) is a Christian saint and martyr.[1] The emperor Decius published a decree ordering the veneration of busts of the deified emperors.")
+      expect(ps.segment).to eq(["Saint Maximus (died 250) is a Christian saint and martyr.[1]", "The emperor Decius published a decree ordering the veneration of busts of the deified emperors."])
+    end
+
+    it 'correctly segments text #124' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Differing agendas can potentially create an understanding gap in a consultation.11 12 Take the example of one of the most common presentations in ill health: the common cold.")
+      expect(ps.segment).to eq(["Differing agendas can potentially create an understanding gap in a consultation.11 12", "Take the example of one of the most common presentations in ill health: the common cold."])
+    end
+
+    it 'correctly segments text #125' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Daniel Kahneman popularised the concept of fast and slow thinking: the distinction between instinctive (type 1 thinking) and reflective, analytical cognition (type 2).10 This model relates to doctors achieving a balance between efficiency and effectiveness.")
+      expect(ps.segment).to eq(["Daniel Kahneman popularised the concept of fast and slow thinking: the distinction between instinctive (type 1 thinking) and reflective, analytical cognition (type 2).10", "This model relates to doctors achieving a balance between efficiency and effectiveness."])
+    end
+
+    it 'correctly segments text #126' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Its traditional use[1] is well documented in the ethnobotanical literature [2–11]. Leaves, buds, tar and essential oils are used to treat a wide spectrum of diseases.")
+      expect(ps.segment).to eq(["Its traditional use[1] is well documented in the ethnobotanical literature [2–11].", "Leaves, buds, tar and essential oils are used to treat a wide spectrum of diseases."])
+    end
+
+    it 'correctly segments text #127' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Thus increasing the desire for political reform both in Lancashire and in the country at large.[7][8] This was a serious misdemeanour,[16] encouraging them to declare the assembly illegal as soon as it was announced on 31 July.[17][18] The radicals sought a second opinion on the meeting's legality.")
+      expect(ps.segment).to eq(["Thus increasing the desire for political reform both in Lancashire and in the country at large.[7][8]", "This was a serious misdemeanour,[16] encouraging them to declare the assembly illegal as soon as it was announced on 31 July.[17][18]", "The radicals sought a second opinion on the meeting's legality."])
+    end
+
+    it 'correctly segments text #128' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "The table in (4) is a sample from the Wall Street Journal (1987).1 According to the distribution all the pairs given in (4) count as candidates for abbreviations.")
+      expect(ps.segment).to eq([ "The table in (4) is a sample from the Wall Street Journal (1987).1", "According to the distribution all the pairs given in (4) count as candidates for abbreviations."])
+    
+    end
   end
 end

data/spec/pragmatic_segmenter/languages/kazakh_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+
+RSpec.describe PragmaticSegmenter::Languages::Kazakh, "(kk)" do
+
+  context "Golden Rules" do
+    it "Simple period to end sentence #001" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Мұхитқа тікелей шыға алмайтын мемлекеттердің ішінде Қазақстан - ең үлкені.", language: "kk")
+      expect(ps.segment).to eq(["Мұхитқа тікелей шыға алмайтын мемлекеттердің ішінде Қазақстан - ең үлкені."])
+    end
+
+    it "Question mark to end sentence #002" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Оқушылар үйі, Достық даңғылы, Абай даналығы, ауыл шаруашылығы – кім? не?", language: "kk")
+      expect(ps.segment).to eq(["Оқушылар үйі, Достық даңғылы, Абай даналығы, ауыл шаруашылығы – кім?", "не?"])
+    end
+
+    it "Parenthetical inside sentence #003" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Әр түрлі өлшемнің атауы болып табылатын м (метр), см (сантиметр), кг (киллограмм), т (тонна), га (гектар), ц (центнер), т. б. (тағы басқа), тәрізді белгілер де қысқарған сөздер болып табылады.", language: "kk")
+      expect(ps.segment).to eq(["Әр түрлі өлшемнің атауы болып табылатын м (метр), см (сантиметр), кг (киллограмм), т (тонна), га (гектар), ц (центнер), т. б. (тағы басқа), тәрізді белгілер де қысқарған сөздер болып табылады."])
+    end
+
+    it "Two letter abbreviation to end sentence #004" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Мысалы: обкомға (облыстық комитетке) барды, ауаткомда (аудандық атқару комитетінде) болды, педучилищеге (педагогтік училищеге) түсті, медпункттің (медициналық пункттің) алдында т. б.", language: "kk")
+      expect(ps.segment).to eq(["Мысалы: обкомға (облыстық комитетке) барды, ауаткомда (аудандық атқару комитетінде) болды, педучилищеге (педагогтік училищеге) түсті, медпункттің (медициналық пункттің) алдында т. б."])
+    end
+
+    it "Number as non sentence boundary #005" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Елдің жалпы ішкі өнімі ЖІӨ (номинал) = $225.619 млрд (2014)", language: "kk")
+      expect(ps.segment).to eq(["Елдің жалпы ішкі өнімі ЖІӨ (номинал) = $225.619 млрд (2014)"])
+    end
+
+    it "No whitespace between sentence boundary #006" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Ресейдiң әлеуметтiк-экономикалық жағдайы.XVIII ғасырдың бiрiншi ширегiнде Ресейге тән нәрсе.", language: "kk")
+      expect(ps.segment).to eq(["Ресейдiң әлеуметтiк-экономикалық жағдайы.", "XVIII ғасырдың бiрiншi ширегiнде Ресейге тән нәрсе."])
+    end
+
+    it "Dates within sentence #007" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "(«Егемен Қазақстан», 7 қыркүйек 2012 жыл. №590-591); Бұл туралы кеше санпедқадағалау комитетінің облыыстық департаменті хабарлады. («Айқын», 23 сəуір 2010 жыл. № 70).", language: "kk")
+      expect(ps.segment).to eq(["(«Егемен Қазақстан», 7 қыркүйек 2012 жыл. №590-591); Бұл туралы кеше санпедқадағалау комитетінің облыыстық департаменті хабарлады.", "(«Айқын», 23 сəуір 2010 жыл. № 70)."])
+    end
+
+    it "Multi period abbreviation within sentence #008" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Иран революциясы (1905 — 11) және азаматтық қозғалыс (1918 — 21) кезінде А. Фарахани, М. Кермани, М. Т. Бехар, т.б. ақындар демократиялық идеяның жыршысы болды.", language: "kk")
+      expect(ps.segment).to eq(["Иран революциясы (1905 — 11) және азаматтық қозғалыс (1918 — 21) кезінде А. Фарахани, М. Кермани, М. Т. Бехар, т.б. ақындар демократиялық идеяның жыршысы болды."])
+    end
+
+    it "Web addresses #009" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Владимир Федосеев: Аттар магиясы енді жоқ http://www.vremya.ru/2003/179/10/80980.html", language: "kk")
+      expect(ps.segment).to eq(["Владимир Федосеев: Аттар магиясы енді жоқ http://www.vremya.ru/2003/179/10/80980.html"])
+    end
+
+    it "Question mark not at end of sentence #010" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Бірақ оның енді не керегі бар? — деді.", language: "kk")
+      expect(ps.segment).to eq(["Бірақ оның енді не керегі бар? — деді."])
+    end
+
+    it "Exclamation mark not at end of sentence #011" do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Сондықтан шапаныма жегізіп отырғаным! - деп, жауап береді.", language: "kk")
+      expect(ps.segment).to eq(["Сондықтан шапаныма жегізіп отырғаным! - деп, жауап береді."])
+    end
+  end
+
+  describe '#segment' do
+    it 'correctly segments text #001' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "Б.з.б. 6 – 3 ғасырларда конфуцийшілдік, моизм, легизм мектептерінің қалыптасуы нәтижесінде Қытай философиясы пайда болды.", language: 'kk')
+      expect(ps.segment).to eq(["Б.з.б. 6 – 3 ғасырларда конфуцийшілдік, моизм, легизм мектептерінің қалыптасуы нәтижесінде Қытай философиясы пайда болды."])
+    end
+
+    it 'correctly segments text #002' do
+      ps = PragmaticSegmenter::Segmenter.new(text: "'Та марбута' тек сөз соңында екі түрде жазылады:", language: "kk")
+      expect(ps.segment).to eq(["'Та марбута' тек сөз соңында екі түрде жазылады:"])
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.3.18
+  version: 0.3.23
 platform: ruby
 authors:
 - Kevin S. Dias
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-03-27 00:00:00.000000000 Z
+date: 2021-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode
@@ -28,30 +28,30 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -124,6 +124,7 @@ files:
 - lib/pragmatic_segmenter/languages/hindi.rb
 - lib/pragmatic_segmenter/languages/italian.rb
 - lib/pragmatic_segmenter/languages/japanese.rb
+- lib/pragmatic_segmenter/languages/kazakh.rb
 - lib/pragmatic_segmenter/languages/persian.rb
 - lib/pragmatic_segmenter/languages/polish.rb
 - lib/pragmatic_segmenter/languages/russian.rb
@@ -152,6 +153,7 @@ files:
 - spec/pragmatic_segmenter/languages/hindi_spec.rb
 - spec/pragmatic_segmenter/languages/italian_spec.rb
 - spec/pragmatic_segmenter/languages/japanese_spec.rb
+- spec/pragmatic_segmenter/languages/kazakh_spec.rb
 - spec/pragmatic_segmenter/languages/persian_spec.rb
 - spec/pragmatic_segmenter/languages/polish_spec.rb
 - spec/pragmatic_segmenter/languages/russian_spec.rb
@@ -164,7 +166,7 @@ homepage: https://github.com/diasks2/pragmatic_segmenter
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -179,9 +181,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.1
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.7.6
+signing_key:
 specification_version: 4
 summary: A rule-based sentence boundary detection gem that works out-of-the-box across
   many languages
@@ -202,6 +204,7 @@ test_files:
 - spec/pragmatic_segmenter/languages/hindi_spec.rb
 - spec/pragmatic_segmenter/languages/italian_spec.rb
 - spec/pragmatic_segmenter/languages/japanese_spec.rb
+- spec/pragmatic_segmenter/languages/kazakh_spec.rb
 - spec/pragmatic_segmenter/languages/persian_spec.rb
 - spec/pragmatic_segmenter/languages/polish_spec.rb
 - spec/pragmatic_segmenter/languages/russian_spec.rb