pragmatic_tokenizer 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0feac0e3ed363c652e85fda4dd631b6f221da44af13ef9da24e9f0be5a0cdd38
-  data.tar.gz: adc17ceadcf5749c1c9d1abce01f28658511a9bd251e25dda2ccf333f32cd355
+  metadata.gz: 15de4932ae8a9e1d96e42552acb4b8247737fc0b343dab265b5e50ca6e5d9d78
+  data.tar.gz: 378e5853911490a38b0fbec242c7e1a95b9e8d4f3499ac59633ea7494d90a127
 SHA512:
-  metadata.gz: 99bbf93e63cf667703c2c386e51592873f9c70fa20df823d15ef69cf3714fef98c48d2428d451ce9c1433c3b1d23c5a317a2ecf1fc5ed6f9be04a66c8d268773
-  data.tar.gz: ddb68cee3ea6cf0b4d2bbe581c99a37d1ecb34de5a2c0703073cc53b27d6520dbf6d9bdd811f0bfcb244120c91290040f269eefeb002c3440d8c9fa55a2d9671
+  metadata.gz: 9ae171101502bc657e8a3c61a0fdf7f22d8aae99957ddebacc0048327882f4f3eb1435bc67b566c0550a36cc0eb818cd9a6f96c393554d2a06f1212cb86f1bd8
+  data.tar.gz: 7158572d7496732af4004c5684373df22786c0bdb13564c7a0865a20394d2d750191b84d42e8025c68e11eb1143c60893f92ab7bb251bbe1d9fa4918482eedb7

data/lib/pragmatic_tokenizer/pre_processor.rb

@@ -17,7 +17,7 @@ module PragmaticTokenizer
     private
 
       def remove_non_breaking_space!
-        gsub!(Regex::NO_BREAK_SPACE, ''.freeze)
+        gsub!(Regex::NO_BREAK_SPACE, ' '.freeze)
       end
 
       def shift_various_characters!

data/lib/pragmatic_tokenizer/regex.rb

@@ -27,6 +27,7 @@ module PragmaticTokenizer
     ASTERISK                      = /(?:\*+)/
     UNDERSCORE                    = /(?:_+)/
     HYPHEN_OR_UNDERSCORE          = /(?:[-_])/
+    LONG_WORD_SPLIT               = /(?:[-_\/—–])/
     PERIOD_AND_PRIOR              = /(?:(.+\.))/
     PERIOD_ONLY                   = /(?:(\.))/
     CONTRACTIONS                  = /(?:[‘’‚‛‹›＇´`])/

data/lib/pragmatic_tokenizer/tokenizer.rb

@@ -253,7 +253,7 @@ module PragmaticTokenizer
         return token if token.length <= @long_word_split
         return token if token =~ Regex::ONLY_HASHTAG_MENTION
         return token if token =~ Regex::DOMAIN_OR_EMAIL
-        token.split(Regex::HYPHEN_OR_UNDERSCORE)
+        token.split(Regex::LONG_WORD_SPLIT)
       end
 
       def chosen_case(text)

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "3.1.0".freeze
+  VERSION = "3.2.0".freeze
 end

data/spec/languages/english_spec.rb

@@ -88,6 +88,12 @@ describe PragmaticTokenizer do
           expect(pt.tokenize(text)).to eq(["#ab-cd"])
         end
 
+        it 'tokenizes a string #015' do
+          text = "In 2004, he co-founded Palantir Technologies, which offers platforms for finance companies and intelligence, defense, and law enforcement communities to integrate, visualize, and analyze the world's information."
+          pt = PragmaticTokenizer::Tokenizer.new
+          expect(pt.tokenize(text)).to eq(["in", "2004", ",", "he", "co-founded", "palantir", "technologies", ",", "which", "offers", "platforms", "for", "finance", "companies", "and", "intelligence", ",", "defense", ",", "and", "law", "enforcement", "communities", "to", "integrate", ",", "visualize", ",", "and", "analyze", "the", "world's", "information", "."])
+        end
+
         it 'handles numbers with symbols 2' do
           text = "Pittsburgh Steelers won 18:16 against Cincinnati Bengals!"
           pt = PragmaticTokenizer::Tokenizer.new
@@ -543,6 +549,13 @@ describe PragmaticTokenizer do
           )
           expect(pt.tokenize(text)).to eq(["some", "main", "categories", "of", "the", "mathematics", "test", "have", "sub", "examples", "that", "most", "14", "year", "olds", "can't", "answer", ",", "therefor", "the", "implementation", "instruction", "made", "in", "the", "1990", "years", "needs", "to", "be", "revised", "."])
         end
+        it 'tokenizes something with a slash' do
+          text = "EO/AA/M/F/Veterans/Disability/Sexual Orientation/Gender Identity"
+          pt = PragmaticTokenizer::Tokenizer.new(
+              long_word_split: 1
+          )
+          expect(pt.tokenize(text)).to eq(["eo", "aa", "m", "f", "veterans", "disability", "sexual", "orientation", "gender", "identity"])
+        end
       end
 
       context 'option (clean)' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - Kevin S. Dias
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-10 00:00:00.000000000 Z
+date: 2020-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode
@@ -153,7 +153,7 @@ files:
 homepage: https://github.com/diasks2/pragmatic_tokenizer
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -168,9 +168,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
+rubyforge_project:
 rubygems_version: 2.7.6
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A multilingual tokenizer
 test_files: