pristine_text 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 26c345b29f87af15925158a92b87951a401ccd1e
-  data.tar.gz: f6bfa2f8ac0574daa87f37eb2f5634a3656686fa
+  metadata.gz: aaae999b5981fa51e5d31efeaf45c35e061b3e46
+  data.tar.gz: 345716e8f916ae88766fed3f342690ea7f8a18ff
 SHA512:
-  metadata.gz: 124f737e2ed1d18e423fae1d12033767d657cb7b9c5281472baf2736e096efc41077b6515087a6498bae0114e861b48216924b365156fdb55d571f821fa01ec4
-  data.tar.gz: 7d9320dd6e6443b376af26bcc74ef83812ef60f881281216e90358d2dfa301bc97fd90a026004d2028cd13934b96f2ab7a61d2434285bd9c555431fa5d8ac234
+  metadata.gz: 22f76bfcc47e5233a13ded3e9c82304c19c8b8656e02fd4e184e37ad0a3c12b95471b8fa3b877202659fe69a75c791e6f56f1019fed5b66dcbebb8cf966d6b7f
+  data.tar.gz: 99a2c9034af5efea3c4be578aecab883553350c210c6df78e1d54bbb3e28499c21b7a55dc7b0b1dd306be1640d2c2739f987f62c1283455382005bdefd29a563

data/lib/pristine_text.rb

@@ -1,7 +1,7 @@
 require "pristine_text/version"
 require "open3"
 require "unicode_utils"
-require "cgi"
+require "htmlentities"
 
 module PristineText
   def self.pipe(text, locale)
@@ -24,7 +24,7 @@ module PristineText
   end
 
   def self.clean(text, locale= :en, stem= true)
-    text= UnicodeUtils.downcase(CGI.unescapeHTML(text).gsub(/ /, "\n"), locale).
+    text= UnicodeUtils.downcase(HTMLEntities.new.decode(text), locale).
       gsub(/[^\p{Letter}\s]+/, "").
       strip.squeeze
     if stem

data/lib/pristine_text/version.rb

@@ -1,3 +1,3 @@
 module PristineText
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/pristine_text.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_dependency 'unicode_utils', '~> 0'
+  spec.add_dependency 'unicode_utils', '~> 1.4'
+  spec.add_dependency 'htmlentities', '~> 4.3'
   spec.required_ruby_version= '~> 2.1'
 end

data/test/pristine_text.rb

@@ -1,9 +1,10 @@
+#encoding: UTF-8
 require "pristine_text"
 require "minitest/autorun"
 
 class PristineTextTest< Minitest::Unit::TestCase
   def test_clean
     assert_equal PristineText.clean("haberler geliyorlar gidiyorlar", :tr), "haber geliyor gidiyor"
-    assert_equal PristineText.clean("}{ÜĞ09&nbsp;İŞi!'^+çö\n\t\v][';.,üğişçö&quot;", :tr), "üğ işiçö üğişçö"
+    assert_equal PristineText.clean("}{ÜĞ09&nbsp;İŞi!'^+çö\n\t\v][';.,üğişçö&quot;", :tr), "üğişiçö üğişçö"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pristine_text
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Nurettin Onur TUĞCU
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-01 00:00:00.000000000 Z
+date: 2014-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
 description: This gem uses unicode_utils to lowercase text, removes non-letters, strips
   and squeezes whitespace, then optionally uses stemwords (from libstemming-tools)
   to stem every word.