pragmatic_tokenizer 2.2.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee933fd568e4ccb0af6034488d9bd0ea15288d25
-  data.tar.gz: 90acbd94ecf4fb8f0ce3671cd1136f8ce309d676
+  metadata.gz: 241faea11370fc685c55a22eae88d9af30fa955c
+  data.tar.gz: 2daa6aa5bae004836538b4bd632067074782a87b
 SHA512:
-  metadata.gz: 896110659d02729735f16d9c740573c4cfb3367021d3e248d8e8421d64440d8f24c8a3020273f026eb5f7dfcb007f9e3ffa14f298c39988f0477b9142cbd9829
-  data.tar.gz: 69ca7908471f37a03ed9953f547fb36043d43c351b29b49c73995bfdcec361f6edaa2a67001a79b8e33b8c4b80d9d7f3b02277a1174b7bc28ebfe8a1715473b0
+  metadata.gz: 54f9fb11af6e42f4e35d6a878dae45e5fd0850793671ae7023dd8f8e17f7e307625f9b8497c5912b8c14312235150ad2cd19cf7f15fa693ac7cee427827677ef
+  data.tar.gz: 2ea45b90bfc8df8044ebab404e89e0936ce391e0d7e5206fc60e775ca65cb67af0fabe3ec787855b359ac0712bdece0eaefdb2dc9bcb0635c41fc0ad604a1bdf

data/lib/pragmatic_tokenizer/full_stop_separator.rb

@@ -23,7 +23,7 @@ module PragmaticTokenizer
           if downcase
             abbreviation = abbr[w]
           else
-            abbreviation = abbr[UnicodeCaseConverter::downcase(w)]
+            abbreviation = abbr[Unicode.downcase(w)]
           end
           unless abbreviation || w =~ /\A[a-z]\z/i ||
                  w =~ /[a-z](?:\.[a-z])+\z/i
@@ -35,11 +35,11 @@ module PragmaticTokenizer
         cleaned_tokens << tokens[i]
       end
       if downcase
-        abbr_included = abbreviations.include?(cleaned_tokens[-1].chomp(".")) unless cleaned_tokens[-1].nil?
+        abbreviation = abbreviations.include?(cleaned_tokens[-1].chomp(".")) unless cleaned_tokens[-1].nil?
       else
-        abbr_included = abbreviations.include?(UnicodeCaseConverter::downcase(cleaned_tokens[-1]).chomp(".")) unless cleaned_tokens[-1].nil?
+        abbreviation = abbreviations.include?(Unicode.downcase(cleaned_tokens[-1]).chomp(".")) unless cleaned_tokens[-1].nil?
       end
-      if cleaned_tokens[-1] && cleaned_tokens[-1] =~ /\A(.*\w)\.\z/ && !abbr_included
+      if cleaned_tokens[-1] && cleaned_tokens[-1] =~ /\A(.*\w)\.\z/ && !abbreviation
         cleaned_tokens[-1] = Regexp.last_match(1)
         cleaned_tokens.push '.'
       end

data/lib/pragmatic_tokenizer/post_processor.rb

@@ -94,7 +94,7 @@ module PragmaticTokenizer
         if downcase
           token.split(/(\.)/)[0]
         else
-          UnicodeCaseConverter::downcase(token.split(/(\.)/)[0])
+          Unicode.downcase(token.split(/(\.)/)[0])
         end
       end
 

data/lib/pragmatic_tokenizer/pre_processor.rb

@@ -33,7 +33,7 @@ module PragmaticTokenizer
     private
 
       def remove_non_breaking_space!
-        gsub!(/\u{00A0}/, ''.freeze)
+         gsub!(/\u{00A0}/, ''.freeze)
       end
 
       # Shift commas off everything but numbers

data/lib/pragmatic_tokenizer/tokenizer.rb

@@ -1,12 +1,10 @@
 # -*- encoding : utf-8 -*-
-require 'set'
-require 'cgi'
 require 'pragmatic_tokenizer/languages'
 require 'pragmatic_tokenizer/pre_processor'
 require 'pragmatic_tokenizer/post_processor'
 require 'pragmatic_tokenizer/full_stop_separator'
 require 'pragmatic_tokenizer/ending_punctuation_separator'
-require 'unicode_case_converter'
+require 'unicode'
 
 module PragmaticTokenizer
   class Tokenizer
@@ -193,7 +191,7 @@ module PragmaticTokenizer
         normalized = inverse_case(token.gsub(REGEX_CONTRACTIONS, "'".freeze))
         return token unless @contractions.key?(normalized)
         result    = @contractions[normalized].split(SPACE_STRING)
-        result[0] = UnicodeCaseConverter::capitalize(result[0]) unless @downcase
+        result[0] = Unicode.capitalize(result[0]) unless @downcase
         result
       end
 
@@ -292,12 +290,12 @@ module PragmaticTokenizer
       end
 
       def chosen_case(token)
-        @downcase ? UnicodeCaseConverter::downcase(token) : token
+        @downcase ? Unicode.downcase(token) : token
       end
 
       def inverse_case(token)
-        @downcase ? token : UnicodeCaseConverter::downcase(token)
+        @downcase ? token : Unicode.downcase(token)
       end
 
   end
-end
+end

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "2.2.1".freeze
+  VERSION = "3.0.0".freeze
 end

data/pragmatic_tokenizer.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "unicode_case_converter", "~> 1.0"
+  spec.add_runtime_dependency "unicode"
   spec.add_development_dependency "bundler", "~> 1.9"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"

data/spec/languages/english_spec.rb

@@ -1335,27 +1335,6 @@ describe PragmaticTokenizer do
           )
           expect(pt.tokenize(text)).to eq(["user", "john", "pt-br", "wordfast"])
         end
-
-        it 'removes non-breaking spaces' do
-          text = "%20141201~221624  %User ID,JU,JU John %TU=00000362  %PT-BR  %Wordfast    da hello."
-          pt = PragmaticTokenizer::Tokenizer.new(
-            language: :en,
-            filter_languages: [:en],
-            clean: true,
-            numbers: :none,
-            minimum_length: 3,
-            expand_contractions: true,
-            remove_stop_words: true,
-            punctuation: :none,
-            remove_emails: true,
-            remove_domains: true,
-            remove_urls: true,
-            hashtags: :remove,
-            mentions: :remove,
-            downcase: true
-          )
-          expect(pt.tokenize(text)).to eq(["user", "john", "pt-br", "wordfast"])
-        end
       end
     end
 

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 2.2.1
+  version: 3.0.0
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-16 00:00:00.000000000 Z
+date: 2016-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: unicode_case_converter
+  name: unicode
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement