pragmatic_tokenizer 3.1.0 → 3.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0feac0e3ed363c652e85fda4dd631b6f221da44af13ef9da24e9f0be5a0cdd38
-  data.tar.gz: adc17ceadcf5749c1c9d1abce01f28658511a9bd251e25dda2ccf333f32cd355
+  metadata.gz: e65a28fbe1b06c5aaed25f8fc5bce8b46b2a8dbcaeddee07139ae733fe9ed51a
+  data.tar.gz: 767847d1d88d74c645763fcc569060e4e3c5beeb8e41364f437562eaeb796f08
 SHA512:
-  metadata.gz: 99bbf93e63cf667703c2c386e51592873f9c70fa20df823d15ef69cf3714fef98c48d2428d451ce9c1433c3b1d23c5a317a2ecf1fc5ed6f9be04a66c8d268773
-  data.tar.gz: ddb68cee3ea6cf0b4d2bbe581c99a37d1ecb34de5a2c0703073cc53b27d6520dbf6d9bdd811f0bfcb244120c91290040f269eefeb002c3440d8c9fa55a2d9671
+  metadata.gz: 1a88cf9354785dbd50890a088f0251eac8ea817c915b46afdf9bc82a7d47a1f60522761b23144861ffb5ebb4bf7a516a4e00a60a4e0d3af621f7618bb5c95ad7
+  data.tar.gz: 8afec02448e6552d5d84b9d296f69185d864c1617eb5643304a5e74132aa1b6d0262d3e2ca1f88bc682af2bb39320618ae7ae8a320b0617ac1b4883d6801909f

data/lib/pragmatic_tokenizer/pre_processor.rb

@@ -17,7 +17,7 @@ module PragmaticTokenizer
     private
 
       def remove_non_breaking_space!
-        gsub!(Regex::NO_BREAK_SPACE, ''.freeze)
+        gsub!(Regex::NO_BREAK_SPACE, ' '.freeze)
       end
 
       def shift_various_characters!

data/lib/pragmatic_tokenizer/regex.rb

@@ -27,6 +27,7 @@ module PragmaticTokenizer
     ASTERISK                      = /(?:\*+)/
     UNDERSCORE                    = /(?:_+)/
     HYPHEN_OR_UNDERSCORE          = /(?:[-_])/
+    LONG_WORD_SPLIT               = /(?:[-_\/—–])/
     PERIOD_AND_PRIOR              = /(?:(.+\.))/
     PERIOD_ONLY                   = /(?:(\.))/
     CONTRACTIONS                  = /(?:[‘’‚‛‹›＇´`])/

data/lib/pragmatic_tokenizer/tokenizer.rb

@@ -253,7 +253,7 @@ module PragmaticTokenizer
         return token if token.length <= @long_word_split
         return token if token =~ Regex::ONLY_HASHTAG_MENTION
         return token if token =~ Regex::DOMAIN_OR_EMAIL
-        token.split(Regex::HYPHEN_OR_UNDERSCORE)
+        token.split(Regex::LONG_WORD_SPLIT)
       end
 
       def chosen_case(text)

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "3.1.0".freeze
+  VERSION = "3.2.1".freeze
 end

data/lib/unicode.rb

@@ -0,0 +1,9 @@
+module Unicode
+  def self.downcase(text)
+    text.downcase
+  end
+
+  def self.capitalize(text)
+    text.capitalize
+  end
+end

data/pragmatic_tokenizer.gemspec

@@ -17,7 +17,6 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "unicode"
   spec.add_development_dependency "bundler", "~> 1.9"
   spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec"

data/spec/languages/english_spec.rb

@@ -88,6 +88,12 @@ describe PragmaticTokenizer do
           expect(pt.tokenize(text)).to eq(["#ab-cd"])
         end
 
+        it 'tokenizes a string #015' do
+          text = "In 2004, he co-founded Palantir Technologies, which offers platforms for finance companies and intelligence, defense, and law enforcement communities to integrate, visualize, and analyze the world's information."
+          pt = PragmaticTokenizer::Tokenizer.new
+          expect(pt.tokenize(text)).to eq(["in", "2004", ",", "he", "co-founded", "palantir", "technologies", ",", "which", "offers", "platforms", "for", "finance", "companies", "and", "intelligence", ",", "defense", ",", "and", "law", "enforcement", "communities", "to", "integrate", ",", "visualize", ",", "and", "analyze", "the", "world's", "information", "."])
+        end
+
         it 'handles numbers with symbols 2' do
           text = "Pittsburgh Steelers won 18:16 against Cincinnati Bengals!"
           pt = PragmaticTokenizer::Tokenizer.new
@@ -543,6 +549,13 @@ describe PragmaticTokenizer do
           )
           expect(pt.tokenize(text)).to eq(["some", "main", "categories", "of", "the", "mathematics", "test", "have", "sub", "examples", "that", "most", "14", "year", "olds", "can't", "answer", ",", "therefor", "the", "implementation", "instruction", "made", "in", "the", "1990", "years", "needs", "to", "be", "revised", "."])
         end
+        it 'tokenizes something with a slash' do
+          text = "EO/AA/M/F/Veterans/Disability/Sexual Orientation/Gender Identity"
+          pt = PragmaticTokenizer::Tokenizer.new(
+              long_word_split: 1
+          )
+          expect(pt.tokenize(text)).to eq(["eo", "aa", "m", "f", "veterans", "disability", "sexual", "orientation", "gender", "identity"])
+        end
       end
 
       context 'option (clean)' do

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.1
 platform: ruby
 authors:
 - Kevin S. Dias
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-10 00:00:00.000000000 Z
+date: 2024-08-11 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: unicode
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -142,6 +128,7 @@ files:
 - lib/pragmatic_tokenizer/regex.rb
 - lib/pragmatic_tokenizer/tokenizer.rb
 - lib/pragmatic_tokenizer/version.rb
+- lib/unicode.rb
 - pragmatic_tokenizer.gemspec
 - spec/languages/bulgarian_spec.rb
 - spec/languages/deutsch_spec.rb
@@ -153,7 +140,7 @@ files:
 homepage: https://github.com/diasks2/pragmatic_tokenizer
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -168,9 +155,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.3.26
+signing_key:
 specification_version: 4
 summary: A multilingual tokenizer
 test_files: