pristine_text 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 03ce1c942c7e32460cf0c312a5d897a205f4a2a7
-  data.tar.gz: 67916011e5695da1ec0c521e5f693215f3c82238
+  metadata.gz: 26c345b29f87af15925158a92b87951a401ccd1e
+  data.tar.gz: f6bfa2f8ac0574daa87f37eb2f5634a3656686fa
 SHA512:
-  metadata.gz: c43004bf30849ab031a6c314a52bad0b863c1591304c082e5761dccc55f0bf658e9ea855a6a46e03c6e35412355c62347c494c2b4f8f25ec30a21a2dce10d830
-  data.tar.gz: 43a0bbcb713ab367ccd531448a82a101de22af9f85fe6fdd5acf2556918533cb647ad575f2af790139cdfe89a62f1408486f124d675e236e0129ae886522d015
+  metadata.gz: 124f737e2ed1d18e423fae1d12033767d657cb7b9c5281472baf2736e096efc41077b6515087a6498bae0114e861b48216924b365156fdb55d571f821fa01ec4
+  data.tar.gz: 7d9320dd6e6443b376af26bcc74ef83812ef60f881281216e90358d2dfa301bc97fd90a026004d2028cd13934b96f2ab7a61d2434285bd9c555431fa5d8ac234

data/.gitignore

@@ -12,3 +12,4 @@
 *.o
 *.a
 mkmf.log
+*.gem

data/README.md

@@ -1,6 +1,6 @@
 # PristineText
 
-This gem uses unicode_utils to lowercase text, removes non-letters, strips and squeezes whitespace, then optionally stemwords (from stemming-tools) to stem every word.
+This gem uses unicode_utils to lowercase text, removes non-letters, strips and squeezes whitespace, then optionally uses stemwords (from libstemming-tools) to stem every word.
 
 ## Installation
 
@@ -20,9 +20,9 @@ Or install it yourself as:
 
 ## Usage
 
-require "pristine_text"
+    require "pristine_text"
 
-puts PristineText.clean("haberler geliyorlar gidiyorlar", :tr)
+    puts PristineText.clean("haberler geliyorlar gidiyorlar", :tr)
 
 ## Contributing
 

data/lib/pristine_text/version.rb

@@ -1,3 +1,3 @@
 module PristineText
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/pristine_text.rb

@@ -1,6 +1,7 @@
 require "pristine_text/version"
 require "open3"
 require "unicode_utils"
+require "cgi"
 
 module PristineText
   def self.pipe(text, locale)
@@ -23,7 +24,7 @@ module PristineText
   end
 
   def self.clean(text, locale= :en, stem= true)
-    text= UnicodeUtils.downcase(text, locale).
+    text= UnicodeUtils.downcase(CGI.unescapeHTML(text).gsub(/ /, "\n"), locale).
       gsub(/[^\p{Letter}\s]+/, "").
       strip.squeeze
     if stem

data/pristine_text.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Nurettin Onur TUĞCU"]
   spec.email         = ["onurtugcu@gmail.com"]
   spec.summary       = %q{Lowercase, squeeze, stem text.}
-  spec.description   = %q{This gem uses unicode_utils to lowercase text, removes non-letters, strips and squeezes whitespace, then optionally stemwords (from stemming-tools) to stem every word.}
+  spec.description   = %q{This gem uses unicode_utils to lowercase text, removes non-letters, strips and squeezes whitespace, then optionally uses stemwords (from libstemming-tools) to stem every word.}
   spec.homepage      = "https://github.com/nurettin/pristine_text"
   spec.license       = "MIT"
 

data/test/pristine_text.rb

@@ -0,0 +1,9 @@
+require "pristine_text"
+require "minitest/autorun"
+
+class PristineTextTest< Minitest::Unit::TestCase
+  def test_clean
+    assert_equal PristineText.clean("haberler geliyorlar gidiyorlar", :tr), "haber geliyor gidiyor"
+    assert_equal PristineText.clean("}{ÜĞ09&nbsp;İŞi!'^+çö\n\t\v][';.,üğişçö&quot;", :tr), "üğ işiçö üğişçö"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pristine_text
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Nurettin Onur TUĞCU
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-31 00:00:00.000000000 Z
+date: 2014-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -53,8 +53,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: This gem uses unicode_utils to lowercase text, removes non-letters, strips
-  and squeezes whitespace, then optionally stemwords (from stemming-tools) to stem
-  every word.
+  and squeezes whitespace, then optionally uses stemwords (from libstemming-tools)
+  to stem every word.
 email:
 - onurtugcu@gmail.com
 executables: []
@@ -69,6 +69,7 @@ files:
 - lib/pristine_text.rb
 - lib/pristine_text/version.rb
 - pristine_text.gemspec
+- test/pristine_text.rb
 homepage: https://github.com/nurettin/pristine_text
 licenses:
 - MIT
@@ -89,8 +90,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
 summary: Lowercase, squeeze, stem text.
-test_files: []
+test_files:
+- test/pristine_text.rb