data_cleansing 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a24ad3a5780b445ed15310ad8776d89c122747d9
-  data.tar.gz: ab79edb935ae22415b50c51d1e1c7dc60c7a16c5
+  metadata.gz: 8ba846025b7441eb5a93230b7fbd8ebe2a4d88e3
+  data.tar.gz: 4e209fd6ef57540a8b549d06c314ae4caeddbf59
 SHA512:
-  metadata.gz: 032ce480495e7127cd17b4b1bd39630e51573c579fda5b7cb34bb32f1f3cb6509c8e3ebd9a568b88f6f36cb694d7a7dbd7b32aad040fe6f511ce47b3d01fad3f
-  data.tar.gz: b2d7af9ad633ad5c5045c1103129d843fb5453b256cd5c3cbe3590967d950036da39ab8d6940387c36a51f1501c4b348ef5128b64b3c044b7ff01969392b6d5c
+  metadata.gz: 7b464ca76d4c40f4621d86a32cd76bd4bc3e71e8b5eed18ac094ae651a8f0be58772a503fa096c6798b081cf3030363973b0d96cfd2cf45d6497e14a5b2717f1
+  data.tar.gz: e6933049c6200cadb6e398e3d2af8bae641534942a201c6ed7b8a47fc991f7a843d7b2d1b6cbc1c00f14d837f2de887ac6011f23c187fe33be3c6199a1e18cdf

data/lib/data_cleansing/cleaners.rb

@@ -45,7 +45,7 @@ module Cleaners
   DataCleansing.register_cleaner(:remove_non_printable, RemoveNonPrintable)
 
   # Remove HTML Markup
-  module RemoveHTMLMarkup
+  module ReplaceHTMLMarkup
     HTML_MARKUP = Regexp.compile(/&(amp|quot|gt|lt|apos|nbsp);/in)
 
     def self.call(string)
@@ -53,17 +53,17 @@ module Cleaners
 
       string.gsub!(HTML_MARKUP) do |match|
         case match.downcase
-        when 'amp' then
+        when '&' then
           '&'
-        when 'quot' then
+        when '"' then
           '"'
-        when 'gt' then
+        when '>' then
           '>'
-        when 'lt' then
+        when '<' then
           '<'
-        when 'apos' then
+        when '&apos;' then
           "'"
-        when 'nbsp' then
+        when '&nbsp;' then
           ' '
         else
           "&#{match};"
@@ -71,16 +71,25 @@ module Cleaners
       end || string
     end
   end
-  DataCleansing.register_cleaner(:remove_html_markup, RemoveHTMLMarkup)
+  DataCleansing.register_cleaner(:replace_html_markup, ReplaceHTMLMarkup)
 
-  module ReplaceURIChars
+  module UnescapeURI
     def self.call(string)
       return string unless string.is_a?(String)
 
       URI.unescape(string)
     end
   end
-  DataCleansing.register_cleaner(:replace_uri_chars, ReplaceURIChars)
+  DataCleansing.register_cleaner(:unescape_uri, UnescapeURI)
+
+  module EscapeURI
+    def self.call(string)
+      return string unless string.is_a?(String)
+
+      URI.escape(string)
+    end
+  end
+  DataCleansing.register_cleaner(:escape_uri, EscapeURI)
 
   # Compress multiple whitespace to a single space
   module CompressWhitespace
@@ -123,15 +132,39 @@ module Cleaners
   end
   DataCleansing.register_cleaner(:string_to_integer, StringToInteger)
 
+  # Returns [Integer] after removing all non-digit characters, except '.'
+  # Returns nil if no digits are present in the string.
+  module StringToFloat
+    NUMERIC = Regexp.compile(/[^0-9\.]/)
+
+    def self.call(string)
+      return string unless string.is_a?(String)
+
+      # Remove Non-Digit Chars, except for '.'
+      string.gsub!(NUMERIC, '')
+      string.length > 0 ? string.to_f : nil
+    end
+  end
+  DataCleansing.register_cleaner(:string_to_float, StringToFloat)
+
   # Convert a Date to a Time at the end of day for that date (YYYY-MM-DD 23:59:59)
   # Ex: 2015-12-31 becomes 2015-12-31 23:59:59
   # If something other than a Date object is passed in, it just passes through.
-  module DateToTimeAtEndOfDay
-    def self.call(date)
-      return date unless date.kind_of?(Date)
-
-      date.to_time.end_of_day
+  #
+  # Note: Only works if ActiveSupport is also loaded since it defines Time#end_of_day.
+  module EndOfDay
+    def self.call(datetime)
+      case datetime
+      when String
+        Time.parse(datetime).end_of_day
+      when Date
+        datetime.to_time.end_of_day
+      when Time
+        datetime.end_of_day
+      else
+        datetime
+      end
     end
   end
-  DataCleansing.register_cleaner(:date_to_time_at_end_of_day, DateToTimeAtEndOfDay)
+  DataCleansing.register_cleaner(:end_of_day, EndOfDay)
 end

data/lib/data_cleansing/version.rb

@@ -1,3 +1,3 @@
 module DataCleansing
-  VERSION = '0.8.0'
+  VERSION = '0.9.0'
 end

data/test/cleaners_test.rb

@@ -0,0 +1,209 @@
+require_relative 'test_helper'
+require 'active_support/core_ext/time/calculations'
+
+class CleanersTest < Minitest::Test
+  class User
+    include DataCleansing::Cleanse
+
+    attr_accessor :first_name, :last_name, :address1, :address2,
+      :make_this_upper, :clean_non_word, :clean_non_printable,
+      :clean_html, :clean_from_uri, :clean_to_uri, :clean_whitespace,
+      :clean_digits_only, :clean_to_integer, :clean_to_float, :clean_end_of_day
+
+    cleanse :first_name, :last_name, :address1, :address2, cleaner: :strip
+    cleanse :make_this_upper, cleaner: :upcase
+    cleanse :clean_non_word, cleaner: :remove_non_word
+    cleanse :clean_non_printable, cleaner: :remove_non_printable
+    cleanse :clean_html, cleaner: :replace_html_markup
+    cleanse :clean_from_uri, cleaner: :unescape_uri
+    cleanse :clean_to_uri, cleaner: :escape_uri
+    cleanse :clean_whitespace, cleaner: :compress_whitespace
+    cleanse :clean_digits_only, cleaner: :digits_only
+    cleanse :clean_to_integer, cleaner: :string_to_integer
+    cleanse :clean_to_float, cleaner: :string_to_float
+    cleanse :clean_end_of_day, cleaner: :end_of_day
+  end
+
+  describe 'Cleaners' do
+    it '#strip' do
+      user            = User.new
+      user.first_name = '     jack   black     '
+      user.last_name  = "  \n  \t   joe"
+      user.address1   = "joe \n\n   \n  \t\t    "
+      user.address2   = "joe \n\n bloggs  \n  \t\t    "
+      user.cleanse_attributes!
+      assert_equal 'jack   black', user.first_name
+      assert_equal 'joe', user.last_name
+      assert_equal 'joe', user.address1
+      assert_equal "joe \n\n bloggs", user.address2
+    end
+
+    it '#upcase' do
+      user                 = User.new
+      user.make_this_upper = '     jacK   blAck     '
+      user.cleanse_attributes!
+      assert_equal '     JACK   BLACK     ', user.make_this_upper
+    end
+
+    it '#remove_non_word' do
+      user                = User.new
+      user.clean_non_word = "  !@#$%^&*()+=-~`\t\n   jacK   blAck   <>.,/\"':;{][]\|?/\\  "
+      user.cleanse_attributes!
+      assert_equal 'jacKblAck', user.clean_non_word
+    end
+
+    it '#remove_non_printable' do
+      user                     = User.new
+      user.clean_non_printable = "  !@#$%^&*()+=-~`\t\n   jacK   blAck   <>.,/\"':;{][]\|?/\\  "
+      user.cleanse_attributes!
+      assert_equal "  !@#$%^&*()+=-~`   jacK   blAck   <>.,/\"':;{][]\|?/\\  ", user.clean_non_printable
+    end
+
+    describe '#clean_html' do
+      it 'cleans &quot;' do
+        user            = User.new
+        user.clean_html = 'O&quot;Leary'
+        user.cleanse_attributes!
+        assert_equal 'O"Leary', user.clean_html
+      end
+
+      it 'cleans &amp;' do
+        user            = User.new
+        user.clean_html = 'Jim &amp; Candi'
+        user.cleanse_attributes!
+        assert_equal 'Jim & Candi', user.clean_html
+      end
+
+      it 'cleans &gt;' do
+        user            = User.new
+        user.clean_html = '2 &gt; 1'
+        user.cleanse_attributes!
+        assert_equal '2 > 1', user.clean_html
+      end
+
+      it 'cleans &lt;' do
+        user            = User.new
+        user.clean_html = '1 &lt; 2'
+        user.cleanse_attributes!
+        assert_equal '1 < 2', user.clean_html
+      end
+
+      it 'cleans &apos;' do
+        user            = User.new
+        user.clean_html = '1&apos;2'
+        user.cleanse_attributes!
+        assert_equal "1'2", user.clean_html
+      end
+
+      it 'cleans &nbsp;' do
+        user            = User.new
+        user.clean_html = '1&nbsp;2'
+        user.cleanse_attributes!
+        assert_equal "1 2", user.clean_html
+      end
+
+      it 'cleans &AMP;' do
+        user            = User.new
+        user.clean_html = 'Mutt &AMP; Jeff Inc.'
+        user.cleanse_attributes!
+        assert_equal 'Mutt & Jeff Inc.', user.clean_html
+      end
+
+      it 'does not clean &;' do
+        user            = User.new
+        user.clean_html = 'Mutt &; Jeff Inc.'
+        user.cleanse_attributes!
+        assert_equal 'Mutt &; Jeff Inc.', user.clean_html
+      end
+
+      it 'does not clean &blah;' do
+        user            = User.new
+        user.clean_html = '1&blah;2'
+        user.cleanse_attributes!
+        assert_equal '1&blah;2', user.clean_html
+      end
+    end
+
+    describe '#unescape_uri' do
+      it 'converts %20' do
+        user                = User.new
+        user.clean_from_uri = 'Jim%20%20Bob%20'
+        user.cleanse_attributes!
+        assert_equal 'Jim  Bob ', user.clean_from_uri
+      end
+      it 'converts %20 only' do
+        user                = User.new
+        user.clean_from_uri = '%20'
+        user.cleanse_attributes!
+        assert_equal ' ', user.clean_from_uri
+      end
+    end
+
+    describe '#escape_uri' do
+      it 'converts %20' do
+        user              = User.new
+        user.clean_to_uri = 'Jim  Bob '
+        user.cleanse_attributes!
+        assert_equal 'Jim%20%20Bob%20', user.clean_to_uri
+      end
+      it 'converts %20 only' do
+        user              = User.new
+        user.clean_to_uri = ' '
+        user.cleanse_attributes!
+        assert_equal '%20', user.clean_to_uri
+      end
+    end
+
+    describe '#compress_whitespace' do
+      it 'compresses multiple spaces' do
+        user                  = User.new
+        user.clean_whitespace = '    J im  B ob       '
+        user.cleanse_attributes!
+        assert_equal ' J im B ob ', user.clean_whitespace
+      end
+
+      it 'does not compress single spaces' do
+        user                  = User.new
+        user.clean_whitespace = ' Jack Black'
+        user.cleanse_attributes!
+        assert_equal ' Jack Black', user.clean_whitespace
+      end
+
+      it 'compresses newlines and tabs' do
+        user                  = User.new
+        user.clean_whitespace = "  \n\n  J im  B ob  \t\n\t     "
+        user.cleanse_attributes!
+        assert_equal ' J im B ob ', user.clean_whitespace
+      end
+    end
+
+    it '#digits_only' do
+      user                   = User.new
+      user.clean_digits_only = " 1 !@#$%^&*3()+=-~`\t\n   jacK6   blAck   <>.,/\"':;8{][]9\|?/\\  "
+      user.cleanse_attributes!
+      assert_equal '13689', user.clean_digits_only
+    end
+
+    it '#string_to_integer' do
+      user                  = User.new
+      user.clean_to_integer = " 1 !@#$%^&*3()+=-~`\t\n   jacK6   blAck   <>.,/\"':;8{][]9\|?/\\  "
+      user.cleanse_attributes!
+      assert_equal 136, user.clean_to_integer
+    end
+
+    it '#string_to_float' do
+      user                = User.new
+      user.clean_to_float = " 1 !@#$%^&*3()+=-~`\t\n   jacK6   blAck   <>.,/\"':;8{][]9\|?/\\  "
+      user.cleanse_attributes!
+      assert_equal 136.89, user.clean_to_float
+    end
+
+    it '#date_to_time_at_end_of_day' do
+      user                  = User.new
+      user.clean_end_of_day = Time.parse('2016-03-03 14:33:44 +0000')
+      user.cleanse_attributes!
+      assert_equal Time.parse('2016-03-03 23:59:59 +0000').to_i, user.clean_end_of_day.to_i
+    end
+
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_cleansing
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-01 00:00:00.000000000 Z
+date: 2016-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -55,6 +55,7 @@ files:
 - lib/data_cleansing/railtie.rb
 - lib/data_cleansing/version.rb
 - test/active_record_test.rb
+- test/cleaners_test.rb
 - test/ruby_test.rb
 - test/test_db.sqlite3
 - test/test_helper.rb
@@ -84,6 +85,7 @@ specification_version: 4
 summary: Data Cleansing framework for Ruby, Rails, Mongoid and MongoMapper.
 test_files:
 - test/active_record_test.rb
+- test/cleaners_test.rb
 - test/ruby_test.rb
 - test/test_db.sqlite3
 - test/test_helper.rb