data_cleansing 0.9.0 → 1.0.3

data/test/active_record_test.rb

@@ -1,25 +1,26 @@
-require_relative 'test_helper'
-require 'active_record'
+require_relative "test_helper"
+require "active_record"
 
 ActiveRecord::Base.logger = SemanticLogger[ActiveRecord::Base]
 ActiveRecord::Base.configurations = {
-  'test' => {
-    'adapter'  => 'sqlite3',
-    'database' => 'test/test_db.sqlite3',
-    'pool'     => 5,
-    'timeout'  => 5000
+  "test" => {
+    "adapter"  => "sqlite3",
+    "database" => "test/test_db.sqlite3",
+    "pool"     => 5,
+    "timeout"  => 5000
   }
 }
-ActiveRecord::Base.establish_connection('test')
+ActiveRecord::Base.establish_connection(:test)
 
-ActiveRecord::Schema.define :version => 0 do
-  create_table :users, :force => true do |t|
+ActiveRecord::Schema.define version: 0 do
+  create_table :users, force: true do |t|
     t.string  :first_name
     t.string  :last_name
     t.string  :address1
     t.string  :address2
     t.string  :ssn
     t.integer :zip_code
+    t.text    :text
   end
 end
 
@@ -37,13 +38,13 @@ class User < ActiveRecord::Base
   attr_accessor :instance_var
 
   # Use a global cleaner
-  cleanse :first_name, :last_name, :cleaner => :strip
+  cleanse :first_name, :last_name, cleaner: :strip
 
   # Define a once off cleaner
-  cleanse :address1, :address2, :instance_var, :cleaner => Proc.new {|string| "<< #{string.strip} >>"}
+  cleanse :address1, :address2, :instance_var, cleaner: proc { |string| "<< #{string.strip} >>" }
 
   # Custom Zip Code cleaner
-  cleanse :zip_code, :cleaner => :string_to_integer
+  cleanse :zip_code, cleaner: :string_to_integer
 
   # Automatically cleanse data before validation
   before_validation :cleanse_attributes!
@@ -52,19 +53,22 @@ end
 class User2 < ActiveRecord::Base
   include DataCleansing::Cleanse
   # Use the same table as User above
-  self.table_name = 'users'
+  self.table_name = "users"
+
+  serialize :text
 
   # Test :all cleaner. Only works with ActiveRecord Models
-  cleanse :all, :cleaner => [:strip, Proc.new{|s| "@#{s}@"}], :except => [:address1, :zip_code]
+  # Must explicitly excelude :text since it is serialized
+  cleanse :all, cleaner: [:strip, proc { |s| "@#{s}@" }], except: %i[address1 zip_code text]
 
   # Clean :first_name multiple times
-  cleanse :first_name, :cleaner => Proc.new {|string| "<< #{string} >>"}
+  cleanse :first_name, cleaner: proc { |string| "<< #{string} >>" }
 
   # Clean :first_name multiple times
-  cleanse :first_name, :cleaner => Proc.new {|string| "$#{string}$"}
+  cleanse :first_name, cleaner: proc { |string| "$#{string}$" }
 
   # Custom Zip Code cleaner
-  cleanse :zip_code, :cleaner => :string_to_integer
+  cleanse :zip_code, cleaner: :string_to_integer
 
   # Automatically cleanse data before validation
   before_validation :cleanse_attributes!
@@ -72,82 +76,81 @@ end
 
 class ActiveRecordTest < Minitest::Test
   describe "ActiveRecord Models" do
-
-    it 'have globally registered cleaner' do
+    it "have globally registered cleaner" do
       assert DataCleansing.cleaner(:strip)
     end
 
-    it 'Model.cleanse_attribute' do
-      assert_equal 'joe',                 User.cleanse_attribute(:first_name,   '    joe   ')
-      assert_equal 'black',               User.cleanse_attribute(:last_name,    "\n  black\n")
-      assert_equal '<< 2632 Brown St >>', User.cleanse_attribute(:address1,     "2632 Brown St   \n")
-      assert_equal '<< instance >>',      User.cleanse_attribute(:instance_var, "\n    instance\n\t      ")
-      assert_equal 12345,                 User.cleanse_attribute(:zip_code,     "\n\tblah 12345badtext\n")
+    it "Model.cleanse_attribute" do
+      assert_equal "joe",                 User.cleanse_attribute(:first_name,   "    joe   ")
+      assert_equal "black",               User.cleanse_attribute(:last_name,    "\n  black\n")
+      assert_equal "<< 2632 Brown St >>", User.cleanse_attribute(:address1,     "2632 Brown St   \n")
+      assert_equal "<< instance >>",      User.cleanse_attribute(:instance_var, "\n    instance\n\t      ")
+      assert_equal 12_345, User.cleanse_attribute(:zip_code, "\n\tblah 12345badtext\n")
     end
 
     describe "with user" do
       before do
         @user = User.new(
-          :first_name   => '    joe   ',
-          :last_name    => "\n  black\n",
-          :address1     => "2632 Brown St   \n",
-          :zip_code     => "\n\tblah 12345badtext\n",
-          :instance_var => "\n    instance\n\t      "
+          first_name:   "    joe   ",
+          last_name:    "\n  black\n",
+          address1:     "2632 Brown St   \n",
+          zip_code:     "\n\tblah 12345badtext\n",
+          instance_var: "\n    instance\n\t      "
         )
       end
 
-      it 'only have 3 cleaners' do
+      it "only have 3 cleaners" do
         assert_equal 3, User.send(:data_cleansing_cleaners).size, User.send(:data_cleansing_cleaners)
       end
 
-      it 'cleanse_attributes! using global cleaner' do
+      it "cleanse_attributes! using global cleaner" do
         assert_equal true, @user.valid?
-        assert_equal 'joe', @user.first_name
-        assert_equal 'black', @user.last_name
+        assert_equal "joe", @user.first_name
+        assert_equal "black", @user.last_name
       end
 
-      it 'cleanse_attributes! using attribute specific custom cleaner' do
+      it "cleanse_attributes! using attribute specific custom cleaner" do
         assert_equal true, @user.valid?
-        assert_equal '<< 2632 Brown St >>', @user.address1
-        assert_equal '<< instance >>', @user.instance_var
+        assert_equal "<< 2632 Brown St >>", @user.address1
+        assert_equal "<< instance >>", @user.instance_var
       end
 
-      it 'cleanse_attributes! using global cleaner using rails extensions' do
+      it "cleanse_attributes! using global cleaner using rails extensions" do
         @user.cleanse_attributes!
-        assert_equal 12345, @user.zip_code
+        assert_equal 12_345, @user.zip_code
       end
     end
 
     describe "with user2" do
       before do
         @user = User2.new(
-          :first_name => '    joe   ',
-          :last_name  => "\n  black\n",
-          :ssn        => "\n    123456789   \n  ",
-          :address1   => "2632 Brown St   \n",
-          :zip_code   => "\n\t  blah\n"
+          first_name: "    joe   ",
+          last_name:  "\n  black\n",
+          ssn:        "\n    123456789   \n  ",
+          address1:   "2632 Brown St   \n",
+          zip_code:   "\n\t  blah\n",
+          text:       ["\n    123456789   \n  ", " second "]
         )
       end
 
-      it 'have 4 cleaners defined' do
+      it "have 4 cleaners defined" do
         assert_equal 4, User2.send(:data_cleansing_cleaners).size, User2.send(:data_cleansing_cleaners)
       end
 
-      it 'have 3 attributes cleaners defined' do
+      it "have 3 attributes cleaners defined" do
         # :all, :first_name, :zip_code
         assert_equal 3, User2.send(:data_cleansing_attribute_cleaners).size, User2.send(:data_cleansing_attribute_cleaners)
       end
 
-      it 'cleanse_attributes! clean all attributes' do
+      it "cleanse_attributes! clean all attributes" do
         assert_equal true, @user.valid?
-        assert_equal '$<< @joe@ >>$', @user.first_name, User2.send(:data_cleansing_cleaners)
-        assert_equal '@black@', @user.last_name
+        assert_equal "$<< @joe@ >>$", @user.first_name, User2.send(:data_cleansing_cleaners)
+        assert_equal "@black@", @user.last_name
         assert_equal "2632 Brown St   \n", @user.address1
         assert_equal "@123456789@", @user.ssn
-        assert_equal nil, @user.zip_code, User2.send(:data_cleansing_cleaners)
+        assert_nil @user.zip_code, User2.send(:data_cleansing_cleaners)
+        assert_equal ["\n    123456789   \n  ", " second "], @user.text
       end
-
     end
-
   end
 end

data/test/cleaners_test.rb

@@ -1,14 +1,15 @@
-require_relative 'test_helper'
-require 'active_support/core_ext/time/calculations'
+require_relative "test_helper"
+require "active_support/core_ext/time/calculations"
 
 class CleanersTest < Minitest::Test
   class User
     include DataCleansing::Cleanse
 
     attr_accessor :first_name, :last_name, :address1, :address2,
-      :make_this_upper, :clean_non_word, :clean_non_printable,
-      :clean_html, :clean_from_uri, :clean_to_uri, :clean_whitespace,
-      :clean_digits_only, :clean_to_integer, :clean_to_float, :clean_end_of_day
+                  :make_this_upper, :clean_non_word, :clean_non_printable,
+                  :clean_html, :clean_from_uri, :clean_to_uri, :clean_whitespace,
+                  :clean_digits_only, :clean_to_integer, :clean_to_float, :clean_end_of_day,
+                  :clean_order
 
     cleanse :first_name, :last_name, :address1, :address2, cleaner: :strip
     cleanse :make_this_upper, cleaner: :upcase
@@ -22,188 +23,198 @@ class CleanersTest < Minitest::Test
     cleanse :clean_to_integer, cleaner: :string_to_integer
     cleanse :clean_to_float, cleaner: :string_to_float
     cleanse :clean_end_of_day, cleaner: :end_of_day
+
+    # Call cleaners in the order they are defined
+    cleanse :clean_order, cleaner: %i[upcase strip]
+    cleanse :clean_order, cleaner: ->(val) { val == "BLAH" ? " yes " : " no " }
   end
 
-  describe 'Cleaners' do
-    it '#strip' do
+  describe "Cleaners" do
+    it "#strip" do
       user            = User.new
-      user.first_name = '     jack   black     '
+      user.first_name = "     jack   black     "
       user.last_name  = "  \n  \t   joe"
       user.address1   = "joe \n\n   \n  \t\t    "
       user.address2   = "joe \n\n bloggs  \n  \t\t    "
       user.cleanse_attributes!
-      assert_equal 'jack   black', user.first_name
-      assert_equal 'joe', user.last_name
-      assert_equal 'joe', user.address1
+      assert_equal "jack   black", user.first_name
+      assert_equal "joe", user.last_name
+      assert_equal "joe", user.address1
       assert_equal "joe \n\n bloggs", user.address2
     end
 
-    it '#upcase' do
+    it "#upcase" do
       user                 = User.new
-      user.make_this_upper = '     jacK   blAck     '
+      user.make_this_upper = "     jacK   blAck     "
       user.cleanse_attributes!
-      assert_equal '     JACK   BLACK     ', user.make_this_upper
+      assert_equal "     JACK   BLACK     ", user.make_this_upper
     end
 
-    it '#remove_non_word' do
+    it "#remove_non_word" do
       user                = User.new
       user.clean_non_word = "  !@#$%^&*()+=-~`\t\n   jacK   blAck   <>.,/\"':;{][]\|?/\\  "
       user.cleanse_attributes!
-      assert_equal 'jacKblAck', user.clean_non_word
+      assert_equal "jacKblAck", user.clean_non_word
     end
 
-    it '#remove_non_printable' do
+    it "#remove_non_printable" do
       user                     = User.new
-      user.clean_non_printable = "  !@#$%^&*()+=-~`\t\n   jacK   blAck   <>.,/\"':;{][]\|?/\\  "
+      user.clean_non_printable = "  !@#$%^&*()+=-~`\t\n   jacK   blAck   <>.,/\"':;{][]\|?/\\ \x89 "
       user.cleanse_attributes!
       assert_equal "  !@#$%^&*()+=-~`   jacK   blAck   <>.,/\"':;{][]\|?/\\  ", user.clean_non_printable
     end
 
-    describe '#clean_html' do
-      it 'cleans &quot;' do
+    describe "#clean_html" do
+      it "cleans &quot;" do
         user            = User.new
-        user.clean_html = 'O&quot;Leary'
+        user.clean_html = "O&quot;Leary"
         user.cleanse_attributes!
         assert_equal 'O"Leary', user.clean_html
       end
 
-      it 'cleans &amp;' do
+      it "cleans &amp;" do
         user            = User.new
-        user.clean_html = 'Jim &amp; Candi'
+        user.clean_html = "Jim &amp; Candi"
         user.cleanse_attributes!
-        assert_equal 'Jim & Candi', user.clean_html
+        assert_equal "Jim & Candi", user.clean_html
       end
 
-      it 'cleans &gt;' do
+      it "cleans &gt;" do
         user            = User.new
-        user.clean_html = '2 &gt; 1'
+        user.clean_html = "2 &gt; 1"
         user.cleanse_attributes!
-        assert_equal '2 > 1', user.clean_html
+        assert_equal "2 > 1", user.clean_html
       end
 
-      it 'cleans &lt;' do
+      it "cleans &lt;" do
         user            = User.new
-        user.clean_html = '1 &lt; 2'
+        user.clean_html = "1 &lt; 2"
         user.cleanse_attributes!
-        assert_equal '1 < 2', user.clean_html
+        assert_equal "1 < 2", user.clean_html
       end
 
-      it 'cleans &apos;' do
+      it "cleans &apos;" do
         user            = User.new
-        user.clean_html = '1&apos;2'
+        user.clean_html = "1&apos;2"
         user.cleanse_attributes!
         assert_equal "1'2", user.clean_html
       end
 
-      it 'cleans &nbsp;' do
+      it "cleans &nbsp;" do
         user            = User.new
-        user.clean_html = '1&nbsp;2'
+        user.clean_html = "1&nbsp;2"
         user.cleanse_attributes!
         assert_equal "1 2", user.clean_html
       end
 
-      it 'cleans &AMP;' do
+      it "cleans &AMP;" do
         user            = User.new
-        user.clean_html = 'Mutt &AMP; Jeff Inc.'
+        user.clean_html = "Mutt &AMP; Jeff Inc."
         user.cleanse_attributes!
-        assert_equal 'Mutt & Jeff Inc.', user.clean_html
+        assert_equal "Mutt & Jeff Inc.", user.clean_html
       end
 
-      it 'does not clean &;' do
+      it "does not clean &;" do
         user            = User.new
-        user.clean_html = 'Mutt &; Jeff Inc.'
+        user.clean_html = "Mutt &; Jeff Inc."
         user.cleanse_attributes!
-        assert_equal 'Mutt &; Jeff Inc.', user.clean_html
+        assert_equal "Mutt &; Jeff Inc.", user.clean_html
       end
 
-      it 'does not clean &blah;' do
+      it "does not clean &blah;" do
         user            = User.new
-        user.clean_html = '1&blah;2'
+        user.clean_html = "1&blah;2"
         user.cleanse_attributes!
-        assert_equal '1&blah;2', user.clean_html
+        assert_equal "1&blah;2", user.clean_html
       end
     end
 
-    describe '#unescape_uri' do
-      it 'converts %20' do
+    describe "#unescape_uri" do
+      it "converts %20" do
         user                = User.new
-        user.clean_from_uri = 'Jim%20%20Bob%20'
+        user.clean_from_uri = "Jim%20%20Bob%20"
         user.cleanse_attributes!
-        assert_equal 'Jim  Bob ', user.clean_from_uri
+        assert_equal "Jim  Bob ", user.clean_from_uri
       end
-      it 'converts %20 only' do
+      it "converts %20 only" do
         user                = User.new
-        user.clean_from_uri = '%20'
+        user.clean_from_uri = "%20"
         user.cleanse_attributes!
-        assert_equal ' ', user.clean_from_uri
+        assert_equal " ", user.clean_from_uri
       end
     end
 
-    describe '#escape_uri' do
-      it 'converts %20' do
+    describe "#escape_uri" do
+      it "converts spaces" do
         user              = User.new
-        user.clean_to_uri = 'Jim  Bob '
+        user.clean_to_uri = "Jim  Bob "
         user.cleanse_attributes!
-        assert_equal 'Jim%20%20Bob%20', user.clean_to_uri
+        assert_equal "Jim++Bob+", user.clean_to_uri
       end
-      it 'converts %20 only' do
+      it "converts space only" do
         user              = User.new
-        user.clean_to_uri = ' '
+        user.clean_to_uri = " "
         user.cleanse_attributes!
-        assert_equal '%20', user.clean_to_uri
+        assert_equal "+", user.clean_to_uri
       end
     end
 
-    describe '#compress_whitespace' do
-      it 'compresses multiple spaces' do
+    describe "#compress_whitespace" do
+      it "compresses multiple spaces" do
         user                  = User.new
-        user.clean_whitespace = '    J im  B ob       '
+        user.clean_whitespace = "    J im  B ob       "
         user.cleanse_attributes!
-        assert_equal ' J im B ob ', user.clean_whitespace
+        assert_equal " J im B ob ", user.clean_whitespace
       end
 
-      it 'does not compress single spaces' do
+      it "does not compress single spaces" do
         user                  = User.new
-        user.clean_whitespace = ' Jack Black'
+        user.clean_whitespace = " Jack Black"
         user.cleanse_attributes!
-        assert_equal ' Jack Black', user.clean_whitespace
+        assert_equal " Jack Black", user.clean_whitespace
       end
 
-      it 'compresses newlines and tabs' do
+      it "compresses newlines and tabs" do
         user                  = User.new
         user.clean_whitespace = "  \n\n  J im  B ob  \t\n\t     "
         user.cleanse_attributes!
-        assert_equal ' J im B ob ', user.clean_whitespace
+        assert_equal " J im B ob ", user.clean_whitespace
       end
     end
 
-    it '#digits_only' do
+    it "#digits_only" do
       user                   = User.new
       user.clean_digits_only = " 1 !@#$%^&*3()+=-~`\t\n   jacK6   blAck   <>.,/\"':;8{][]9\|?/\\  "
       user.cleanse_attributes!
-      assert_equal '13689', user.clean_digits_only
+      assert_equal "13689", user.clean_digits_only
     end
 
-    it '#string_to_integer' do
+    it "#string_to_integer" do
       user                  = User.new
       user.clean_to_integer = " 1 !@#$%^&*3()+=-~`\t\n   jacK6   blAck   <>.,/\"':;8{][]9\|?/\\  "
       user.cleanse_attributes!
       assert_equal 136, user.clean_to_integer
     end
 
-    it '#string_to_float' do
+    it "#string_to_float" do
       user                = User.new
       user.clean_to_float = " 1 !@#$%^&*3()+=-~`\t\n   jacK6   blAck   <>.,/\"':;8{][]9\|?/\\  "
       user.cleanse_attributes!
       assert_equal 136.89, user.clean_to_float
     end
 
-    it '#date_to_time_at_end_of_day' do
+    it "#date_to_time_at_end_of_day" do
       user                  = User.new
-      user.clean_end_of_day = Time.parse('2016-03-03 14:33:44 +0000')
+      user.clean_end_of_day = Time.parse("2016-03-03 14:33:44 +0000")
       user.cleanse_attributes!
-      assert_equal Time.parse('2016-03-03 23:59:59 +0000').to_i, user.clean_end_of_day.to_i
+      assert_equal Time.parse("2016-03-03 23:59:59 +0000").to_i, user.clean_end_of_day.to_i
     end
 
+    it "cleans in the order defined" do
+      user             = User.new
+      user.clean_order = "  blah "
+      user.cleanse_attributes!
+      assert_equal " yes ", user.clean_order
+    end
   end
 end

data/test/data_cleansing_test.rb

@@ -0,0 +1,9 @@
+require_relative "test_helper"
+
+class DataCleansingTest < Minitest::Test
+  describe "#clean" do
+    it "can call any cleaner directly" do
+      assert_equal "jack   black", DataCleansing.clean(:strip, "     jack   black     ")
+    end
+  end
+end