text_clean 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 73936fded6a20b6fdde25a06bd5cb74b830177c1
-  data.tar.gz: 7fc77b21e64fad053e1e56f1245ddf70d7b59195
+  metadata.gz: 2e529b66807a01385f10fc896e173880ebe10925
+  data.tar.gz: d4c095af593c650788cc54795ca12a76c695a652
 SHA512:
-  metadata.gz: 596c250db5028aaaaa71c7290e160706e08365b63ba6d9b9aed79ecc03ccea965e6380dfe7709e578fd4ac6f32a733c29d6d07bfdee60a3a41d863bd94bf936f
-  data.tar.gz: 399449bc2bb5c8fafed2a435446488592ad62eb05eda659d1dd4f8338c07280091bcf604f482725b46367b3daa7e1baac73b16530c02d5950f26cfb9f03ca3c3
+  metadata.gz: 8c45e5724b4f8e5598fdeee84869564b737e9ac77ff30c08e756fec9adec5f148f150cf5026dacd0dd16e011357ff02c02d4ceeb61b0a3725400b7e23a98b786
+  data.tar.gz: 15ff835dff5b83c95e71d87e4d483ac54bf9978c8c66d38e67d738b577c0a24a140c5e38528fd5fd0c6af8f60a5c5ddb057b40c43012cc0bf845d37fcd6a1c7d

data/.gitignore

@@ -0,0 +1,5 @@
+Gemfile.lock
+lib/text_clean/text_clean.bundle
+lib/text_clean/text_clean.so
+*.gem
+tmp/

data/Rakefile

@@ -1,3 +1,14 @@
+
+require 'rspec'
+require 'rspec/core/rake_task'
+
+# `rake test`
+desc 'Run all examples'
+RSpec::Core::RakeTask.new(:test) do |t|
+  t.pattern = 'spec/**/*_spec.rb'
+end
+
+# `rake compile`
 require "rake/extensiontask"
 
 Rake::ExtensionTask.new "text_clean" do |ext|

data/ext/text_clean/text_clean.cc

@@ -52,7 +52,7 @@ size_t text_clean_cstr(char* text, long len, char line_sep)
           just_added_period = false;
         }
       } else {
-        // scan ahead to see if this is a hyphen at the end of the line
+        // scan ahead to see if this hyphen is at the end of the line
         char* scan_ahead;
         for (scan_ahead = read + 1; scan_ahead < eos; scan_ahead++) {
           char s = *scan_ahead;
@@ -74,11 +74,11 @@ size_t text_clean_cstr(char* text, long len, char line_sep)
       *write++ = line_sep;
       just_added_period = true;
       just_added_space = false;
-    } else if (c == ' ' && !just_added_space && !just_added_period) {
+    } else if ((c == ' ' || c == '\n') && !just_added_space && !just_added_period) {
       *write++ = ' ';
       just_added_space = true;
       just_added_period = false;
-    } else if (c >= 'a' && c <= 'z') {
+    } else if (c == '\'' || (c >= 'a' && c <= 'z')) {
       *write++ = c;
       just_added_space = false;
       just_added_period = false;

data/lib/text_clean/version.rb

@@ -1,3 +1,3 @@
 module TextClean
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end

data/spec/text_clean_spec.rb

@@ -0,0 +1,61 @@
+require 'rspec'
+require 'text_clean'
+
+def clean(text, line_sep="\n", &block)
+  TextClean.clean(text, line_sep)
+  block.call(text)
+end
+
+def clean_eq(text, expected_after_cleaning, line_sep="\n")
+  clean(text, line_sep) { |t| expect(t).to eq(expected_after_cleaning) }
+end
+
+describe TextClean do
+  it "lowercases text" do
+    clean_eq("ALL UPPERCASE", "all uppercase")
+  end
+
+  it "compacts whitespace" do
+    clean_eq("is\t it    so?", "is it so\n")
+  end
+
+  it "treats [,/&] as whitespace" do
+    clean_eq("a,bb&cc/d", "a bb cc d")
+  end
+
+  it "treats [;:!?] as sentence separators" do
+    clean_eq("x;y?z!:q", "x.y.z.q", ".")
+  end
+
+  it "joins hyphenated words at line end" do
+    clean_eq("satis-\nfaction", "satisfaction")
+  end
+
+  it "ignores whitespace after hyphen at line end" do
+    clean_eq("satis- \t \nfaction", "satisfaction")
+  end
+
+  it "treats a double hyphen as word separator" do
+    clean_eq("good--do it", "good do it")
+  end
+
+  it "compacts whitespace around double hyphen" do
+    clean_eq("good -- do it", "good do it")
+  end
+
+  it "treats newlines as word separator" do
+    clean_eq("a\nb\nc", "a b c")
+  end
+
+  it "ignores numbers" do
+    clean_eq("123abc", "abc")
+  end
+
+  it "ignores quotes and braces" do
+    clean_eq("[jim] \"speaks\"", "jim speaks")
+  end
+
+  it "keeps apostrophes" do
+    clean_eq("dad's", "dad's")
+  end
+end

data/text_clean.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.3"
   spec.add_development_dependency "rake-compiler", "~> 0.9"
   spec.add_development_dependency "byebug", "~> 3.4"
+  spec.add_development_dependency "rspec", "~> 3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: text_clean
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Duane Johnson
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 description: Cleans text by removing punctuation, lowercasing. Very fast.
 email:
 - duane.johnson@gmail.com
@@ -76,6 +90,7 @@ extensions:
 - ext/text_clean/extconf.rb
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - Gemfile
 - Rakefile
 - bin/text_clean
@@ -84,6 +99,7 @@ files:
 - ext/text_clean/text_clean.cc
 - lib/text_clean.rb
 - lib/text_clean/version.rb
+- spec/text_clean_spec.rb
 - text_clean.gemspec
 homepage: https://github.com/wordtreefoundation
 licenses:
@@ -110,4 +126,5 @@ rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Text cleaner
-test_files: []
+test_files:
+- spec/text_clean_spec.rb