data_cleansing 0.9.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8ba846025b7441eb5a93230b7fbd8ebe2a4d88e3
-  data.tar.gz: 4e209fd6ef57540a8b549d06c314ae4caeddbf59
+  metadata.gz: 0347620583101155e6181d7e2504ef2a4816d970
+  data.tar.gz: b80bff6ab7116bda3cef959a7eeb6c231ab3660b
 SHA512:
-  metadata.gz: 7b464ca76d4c40f4621d86a32cd76bd4bc3e71e8b5eed18ac094ae651a8f0be58772a503fa096c6798b081cf3030363973b0d96cfd2cf45d6497e14a5b2717f1
-  data.tar.gz: e6933049c6200cadb6e398e3d2af8bae641534942a201c6ed7b8a47fc991f7a843d7b2d1b6cbc1c00f14d837f2de887ac6011f23c187fe33be3c6199a1e18cdf
+  metadata.gz: 474e0ed54427a7958358a1d645d95792af3b83f0621e48c1986d09ecfd1f8288aada4e6ee55573f88347eb7193adf0eddde1b7cb39c110c6a84a1c5f43daae19
+  data.tar.gz: deb42d04fa24cf7b3e77d8411989e2c8578bf8c23b9f8726898df640e93e271071e0cccd96850eb342780737894823ccd295032a24bb22fdc16678a71b55ddd1

data/README.md

@@ -1,7 +1,7 @@
 data_cleansing
 ==============
 
-Data Cleansing framework for Ruby, Rails, Mongoid and MongoMapper.
+Data Cleansing framework for Ruby.
 
 * http://github.com/reidmorrison/data_cleansing
 
@@ -12,12 +12,8 @@ or trailing blanks and even newlines.
 Similarly it would be useful to be able to attach a cleansing solution to a field
 in a model and have the data cleansed transparently when required.
 
-DataCleansing is a framework that allows any data cleansing to be applied to
-specific attributes or fields. At this time it does not supply the cleaning
-solutions themselves since they are usually straight forward, or so complex
-that they don't tend to be too useful to others. However, over time built-in
-cleansing solutions may be added. Feel free to submit any suggestions via a ticket
-or pull request.
+DataCleansing is a framework that allows data cleansing to be applied to
+specific attributes or fields.
 
 ## Features
 
@@ -297,24 +293,6 @@ Install the Gem with bundler
 
     bundle install
 
-## Architecture
-
-DataCleansing has been designed to support externalized data cleansing routines.
-In this way the data cleansing routine itself can be loaded from a datastore and
-applied dynamically at runtime.
-Although not supported out of the box, this design allows for example for the
-data cleansing routines to be stored in something like [ZooKeeper](http://zookeeper.apache.org/).
-Then any changes to the data cleansing routines can be pushed out immediately to
-every server that needs it.
-
-DataCleansing is designed to support any Ruby model. In this way it can be used
-in just about any ORM or DOM. For example, it currently easily supports both
-Rails and Mongoid models. Some extensions have been added to support these frameworks.
-
-For example, in Rails it obtains the raw data value before Rails has converted it.
-Which is useful for cleansing integer or float fields as raw strings before Rails
-tries to convert it to an integer or float.
-
 ## Dependencies
 
 DataCleansing requires the following dependencies

data/Rakefile

@@ -1,6 +1,4 @@
-require 'rake/clean'
 require 'rake/testtask'
-
 require_relative 'lib/data_cleansing/version'
 
 task :gem do
@@ -14,14 +12,10 @@ task publish: :gem do
   system "rm data_cleansing-#{DataCleansing::VERSION}.gem"
 end
 
-desc 'Run Test Suite'
-task :test do
-  Rake::TestTask.new(:functional) do |t|
-    t.test_files = FileList['test/**/*_test.rb']
-    t.verbose    = true
-  end
-
-  Rake::Task['functional'].invoke
+Rake::TestTask.new(:test) do |t|
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+  t.warning = true
 end
 
 task default: :test

data/lib/data_cleansing/cleaners.rb

@@ -1,4 +1,4 @@
-require 'uri'
+require 'cgi'
 module Cleaners
   # Strip leading and trailing whitespace
   module Strip
@@ -20,6 +20,16 @@ module Cleaners
   end
   DataCleansing.register_cleaner(:upcase, Upcase)
 
+  # Convert to downcase
+  module Downcase
+    def self.call(string)
+      return string unless string.is_a?(String)
+
+      string.downcase! || string
+    end
+  end
+  DataCleansing.register_cleaner(:downcase, Downcase)
+
   # Remove all non-word characters, including whitespace
   module RemoveNonWord
     NOT_WORDS = Regexp.compile(/\W/)
@@ -44,7 +54,7 @@ module Cleaners
   end
   DataCleansing.register_cleaner(:remove_non_printable, RemoveNonPrintable)
 
-  # Remove HTML Markup
+  # Unescape HTML Markup ( case-insensitive )
   module ReplaceHTMLMarkup
     HTML_MARKUP = Regexp.compile(/&(amp|quot|gt|lt|apos|nbsp);/in)
 
@@ -77,7 +87,7 @@ module Cleaners
     def self.call(string)
       return string unless string.is_a?(String)
 
-      URI.unescape(string)
+      CGI.unescape(string)
     end
   end
   DataCleansing.register_cleaner(:unescape_uri, UnescapeURI)
@@ -86,7 +96,7 @@ module Cleaners
     def self.call(string)
       return string unless string.is_a?(String)
 
-      URI.escape(string)
+      CGI.escape(string)
     end
   end
   DataCleansing.register_cleaner(:escape_uri, EscapeURI)

data/lib/data_cleansing/cleanse.rb

@@ -7,10 +7,10 @@ module DataCleansing
     module ClassMethods
       # Define how to cleanse one or more attributes
       def cleanse(*args)
-        last = args.last
+        last       = args.last
         attributes = args.dup
-        params = (last.is_a?(Hash) && last.instance_of?(Hash)) ? attributes.pop.dup : {}
-        cleaners = Array(params.delete(:cleaner))
+        params     = (last.is_a?(Hash) && last.instance_of?(Hash)) ? attributes.pop.dup : {}
+        cleaners   = Array(params.delete(:cleaner))
         raise(ArgumentError, "Mandatory :cleaner parameter is missing: #{params.inspect}") unless cleaners
 
         cleaner = DataCleansingCleaner.new(cleaners, attributes, params)
@@ -58,7 +58,7 @@ module DataCleansing
 
         # Collect parent cleaners first, starting with the top parent
         cleaners = []
-        klass = self
+        klass    = self
         while klass != Object
           if klass.respond_to?(:data_cleansing_attribute_cleaners)
             cleaners += klass.data_cleansing_attribute_cleaners[:all] || []
@@ -66,8 +66,9 @@ module DataCleansing
           end
           klass = klass.superclass
         end
-        cleansed_value = value.dup
-        cleaners.reverse_each {|cleaner| cleansed_value = data_cleansing_clean(cleaner, cleansed_value, object) if cleaner}
+        # Support Fixnum values
+        cleansed_value = value.is_a?(Fixnum) ? value : value.dup
+        cleaners.reverse_each { |cleaner| cleansed_value = data_cleansing_clean(cleaner, cleansed_value, object) if cleaner }
         cleansed_value
       end
 
@@ -90,33 +91,19 @@ module DataCleansing
 
       # Returns the supplied value cleansed using the supplied cleaner
       # Parameters
-      #   object
+      #   binding
       #     If supplied the cleansing will be performed within the scope of
-      #     that object so that cleaners can read and write to attributes
-      #     of that object
+      #     that binding so that cleaners can read and write to attributes
+      #     of that binding
       #
       # No logging of cleansing is performed by this method since the value
       # itself is not modified
-      def data_cleansing_clean(cleaner_struct, value, object=nil)
+      def data_cleansing_clean(cleaner_struct, value, binding = nil)
         return if cleaner_struct.nil? || value.nil?
         # Duplicate value in case cleaner uses methods such as gsub!
         new_value = value.is_a?(String) ? value.dup : value
         cleaner_struct.cleaners.each do |name|
-          # Cleaner itself could be a custom Proc, otherwise do a global lookup for it
-          proc = name.is_a?(Proc) ? name : DataCleansing.cleaner(name.to_sym)
-          raise "No cleaner defined for #{name.inspect}" unless proc
-
-          if proc.is_a?(Proc)
-            new_value = if object
-              # Call the cleaner proc within the scope (binding) of the object
-              proc.arity == 1 ? object.instance_exec(new_value, &proc) : object.instance_exec(new_value, cleaner_struct.params, &proc)
-            else
-              proc.arity == 1 ? proc.call(new_value) : proc.call(new_value, cleaner_struct.params)
-            end
-          else
-            new_value = (proc.method(:call).arity == 1 ? proc.call(new_value) : proc.call(new_value, cleaner_struct.params))
-          end
-
+          new_value = DataCleansing.clean(name, new_value, binding)
         end
         new_value
       end
@@ -135,19 +122,19 @@ module DataCleansing
         changes = {}
         DataCleansing.logger.benchmark_info("#{self.class.name}#cleanse_attributes!", :payload => changes) do
           # Collect parent cleaners first, starting with the top parent
-          cleaners = [self.class.send(:data_cleansing_cleaners)]
+          cleaners       = [self.class.send(:data_cleansing_cleaners)]
           after_cleaners = [self.class.send(:data_cleansing_after_cleaners)]
-          klass = self.class.superclass
+          klass          = self.class.superclass
           while klass != Object
             cleaners << klass.send(:data_cleansing_cleaners) if klass.respond_to?(:data_cleansing_cleaners)
             after_cleaners << klass.send(:data_cleansing_after_cleaners) if klass.respond_to?(:data_cleansing_after_cleaners)
             klass = klass.superclass
           end
           # Capture all modified fields if log_level is :debug or :trace
-          cleaners.reverse_each {|cleaner| changes.merge!(data_cleansing_execute_cleaners(cleaner, verbose))}
+          cleaners.reverse_each { |cleaner| changes.merge!(data_cleansing_execute_cleaners(cleaner, verbose)) }
 
           # Execute the after cleaners, starting with the parent after cleanse methods
-          after_cleaners.reverse_each {|a| a.each {|method| send(method)} }
+          after_cleaners.reverse_each { |a| a.each { |method| send(method) } }
         end
         changes
       end
@@ -176,15 +163,9 @@ module DataCleansing
           # Special case to include :all fields
           # Only works with ActiveRecord based models, not supported with regular Ruby models
           if attrs.include?(:all) && defined?(ActiveRecord) && respond_to?(:attributes)
-            attrs = attributes.keys.collect{|i| i.to_sym}
+            attrs = attributes.keys.collect { |i| i.to_sym }
             attrs.delete(:id)
 
-            # Remove serialized_attributes if any, from the :all condition
-            if self.class.respond_to?(:serialized_attributes)
-              serialized_attrs = self.class.serialized_attributes.keys
-              attrs -= serialized_attrs.collect{|i| i.to_sym} if serialized_attrs
-            end
-
             # Replace any encrypted attributes with their non-encrypted versions if any
             if defined?(SymmetricEncryption) && self.class.respond_to?(:encrypted_attributes)
               self.class.encrypted_attributes.each_pair do |clear, encrypted|
@@ -205,15 +186,16 @@ module DataCleansing
           attrs.each do |attr|
             # Under ActiveModel for Rails and Mongoid need to retrieve raw value
             # before data type conversion
-            value = if respond_to?(:read_attribute_before_type_cast) && has_attribute?(attr.to_s)
-              read_attribute_before_type_cast(attr.to_s)
-            else
-              send(attr.to_sym)
-            end
+            value =
+              if respond_to?(:read_attribute_before_type_cast) && has_attribute?(attr.to_s)
+                read_attribute_before_type_cast(attr.to_s)
+              else
+                send(attr.to_sym)
+              end
 
             # No need to clean if attribute is nil
             unless value.nil?
-              new_value = self.class.send(:data_cleansing_clean,cleaner_struct, value, self)
+              new_value = self.class.send(:data_cleansing_clean, cleaner_struct, value, self)
 
               if new_value != value
                 # Update value only if it has changed
@@ -222,7 +204,7 @@ module DataCleansing
                 # Capture changed attributes
                 if changes
                   # Mask sensitive attributes when logging
-                  masked = DataCleansing.masked_attributes.include?(attr.to_sym)
+                  masked    = DataCleansing.masked_attributes.include?(attr.to_sym)
                   new_value = :masked if masked && !new_value.nil?
                   if previous = changes[attr.to_sym]
                     previous[:after] = new_value
@@ -246,7 +228,7 @@ module DataCleansing
 
     def self.included(base)
       base.class_eval do
-        extend  DataCleansing::Cleanse::ClassMethods
+        extend DataCleansing::Cleanse::ClassMethods
         include DataCleansing::Cleanse::InstanceMethods
       end
     end

data/lib/data_cleansing/data_cleansing.rb

@@ -27,4 +27,22 @@ module DataCleansing
     @@masked_attributes.freeze
   end
 
+  # Run the specified cleanser against the supplied value
+  def self.clean(name, value, binding = nil)
+    # Cleaner itself could be a custom Proc, otherwise do a global lookup for it
+    proc = name.is_a?(Proc) ? name : DataCleansing.cleaner(name.to_sym)
+    raise(ArgumentError, "No cleaner defined for #{name.inspect}") unless proc
+
+    if proc.is_a?(Proc)
+      if binding
+        # Call the cleaner proc within the scope (binding) of the binding
+        proc.arity == 1 ? binding.instance_exec(value, &proc) : binding.instance_exec(value, cleaner_struct.params, &proc)
+      else
+        proc.arity == 1 ? proc.call(value) : proc.call(value, cleaner_struct.params)
+      end
+    else
+      (proc.method(:call).arity == 1 ? proc.call(value) : proc.call(value, cleaner_struct.params))
+    end
+  end
+
 end

data/lib/data_cleansing/version.rb

@@ -1,3 +1,3 @@
 module DataCleansing
-  VERSION = '0.9.0'
+  VERSION = '1.0.0'
 end

data/test/active_record_test.rb

@@ -10,7 +10,7 @@ ActiveRecord::Base.configurations = {
     'timeout'  => 5000
   }
 }
-ActiveRecord::Base.establish_connection('test')
+ActiveRecord::Base.establish_connection(:test)
 
 ActiveRecord::Schema.define :version => 0 do
   create_table :users, :force => true do |t|
@@ -20,6 +20,7 @@ ActiveRecord::Schema.define :version => 0 do
     t.string  :address2
     t.string  :ssn
     t.integer :zip_code
+    t.text    :text
   end
 end
 
@@ -54,8 +55,11 @@ class User2 < ActiveRecord::Base
   # Use the same table as User above
   self.table_name = 'users'
 
+  serialize :text
+
   # Test :all cleaner. Only works with ActiveRecord Models
-  cleanse :all, :cleaner => [:strip, Proc.new{|s| "@#{s}@"}], :except => [:address1, :zip_code]
+  # Must explicitly excelude :text since it is serialized
+  cleanse :all, :cleaner => [:strip, Proc.new{|s| "@#{s}@"}], :except => [:address1, :zip_code, :text]
 
   # Clean :first_name multiple times
   cleanse :first_name, :cleaner => Proc.new {|string| "<< #{string} >>"}
@@ -71,7 +75,7 @@ class User2 < ActiveRecord::Base
 end
 
 class ActiveRecordTest < Minitest::Test
-  describe "ActiveRecord Models" do
+  describe 'ActiveRecord Models' do
 
     it 'have globally registered cleaner' do
       assert DataCleansing.cleaner(:strip)
@@ -118,14 +122,15 @@ class ActiveRecordTest < Minitest::Test
       end
     end
 
-    describe "with user2" do
+    describe 'with user2' do
       before do
         @user = User2.new(
           :first_name => '    joe   ',
           :last_name  => "\n  black\n",
           :ssn        => "\n    123456789   \n  ",
           :address1   => "2632 Brown St   \n",
-          :zip_code   => "\n\t  blah\n"
+          :zip_code   => "\n\t  blah\n",
+          :text       => ["\n    123456789   \n  ", ' second ']
         )
       end
 
@@ -145,6 +150,7 @@ class ActiveRecordTest < Minitest::Test
         assert_equal "2632 Brown St   \n", @user.address1
         assert_equal "@123456789@", @user.ssn
         assert_equal nil, @user.zip_code, User2.send(:data_cleansing_cleaners)
+        assert_equal ["\n    123456789   \n  ", ' second '], @user.text
       end
 
     end

data/test/cleaners_test.rb

@@ -8,7 +8,8 @@ class CleanersTest < Minitest::Test
     attr_accessor :first_name, :last_name, :address1, :address2,
       :make_this_upper, :clean_non_word, :clean_non_printable,
       :clean_html, :clean_from_uri, :clean_to_uri, :clean_whitespace,
-      :clean_digits_only, :clean_to_integer, :clean_to_float, :clean_end_of_day
+      :clean_digits_only, :clean_to_integer, :clean_to_float, :clean_end_of_day,
+      :clean_order
 
     cleanse :first_name, :last_name, :address1, :address2, cleaner: :strip
     cleanse :make_this_upper, cleaner: :upcase
@@ -22,6 +23,10 @@ class CleanersTest < Minitest::Test
     cleanse :clean_to_integer, cleaner: :string_to_integer
     cleanse :clean_to_float, cleaner: :string_to_float
     cleanse :clean_end_of_day, cleaner: :end_of_day
+
+    # Call cleaners in the order they are defined
+    cleanse :clean_order, cleaner: [:upcase, :strip]
+    cleanse :clean_order, cleaner: -> val { val == 'BLAH' ? ' yes ' : ' no ' }
   end
 
   describe 'Cleaners' do
@@ -140,17 +145,17 @@ class CleanersTest < Minitest::Test
     end
 
     describe '#escape_uri' do
-      it 'converts %20' do
+      it 'converts spaces' do
         user              = User.new
         user.clean_to_uri = 'Jim  Bob '
         user.cleanse_attributes!
-        assert_equal 'Jim%20%20Bob%20', user.clean_to_uri
+        assert_equal 'Jim++Bob+', user.clean_to_uri
       end
-      it 'converts %20 only' do
+      it 'converts space only' do
         user              = User.new
         user.clean_to_uri = ' '
         user.cleanse_attributes!
-        assert_equal '%20', user.clean_to_uri
+        assert_equal '+', user.clean_to_uri
       end
     end
 
@@ -205,5 +210,12 @@ class CleanersTest < Minitest::Test
       assert_equal Time.parse('2016-03-03 23:59:59 +0000').to_i, user.clean_end_of_day.to_i
     end
 
+    it 'cleans in the order defined' do
+      user             = User.new
+      user.clean_order = '  blah '
+      user.cleanse_attributes!
+      assert_equal ' yes ', user.clean_order
+    end
+
   end
 end

data/test/data_cleansing_test.rb

@@ -0,0 +1,9 @@
+require_relative 'test_helper'
+
+class DataCleansingTest < Minitest::Test
+  describe '#clean' do
+    it 'can call any cleaner directly' do
+      assert_equal 'jack   black', DataCleansing.clean(:strip, '     jack   black     ')
+    end
+  end
+end

data/test/ruby_test.rb

@@ -75,6 +75,7 @@ class RubyTest < Minitest::Test
       assert_equal 'joe',                 RubyUserChild.cleanse_attribute(:first_name, '    joe   '), RubyUserChild.send(:data_cleansing_attribute_cleaners)
       assert_equal 'black',               RubyUserChild.cleanse_attribute(:last_name,  "\n  black\n"), RubyUserChild.send(:data_cleansing_attribute_cleaners)
       assert_equal '<< 2632 Brown St >>', RubyUserChild.cleanse_attribute(:address1,   "2632 Brown St   \n"), RubyUserChild.send(:data_cleansing_attribute_cleaners)
+      assert_equal 3,                     RubyUserChild.cleanse_attribute(:first_name, 3), RubyUserChild.send(:data_cleansing_attribute_cleaners)
     end
 
     describe "with ruby user" do

data/test/test_helper.rb

@@ -3,7 +3,6 @@ $LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
 require 'yaml'
 require 'minitest/autorun'
 require 'minitest/reporters'
-require 'minitest/stub_any_instance'
 require 'awesome_print'
 require 'data_cleansing'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_cleansing
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-03 00:00:00.000000000 Z
+date: 2016-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -56,6 +56,7 @@ files:
 - lib/data_cleansing/version.rb
 - test/active_record_test.rb
 - test/cleaners_test.rb
+- test/data_cleansing_test.rb
 - test/ruby_test.rb
 - test/test_db.sqlite3
 - test/test_helper.rb
@@ -86,6 +87,7 @@ summary: Data Cleansing framework for Ruby, Rails, Mongoid and MongoMapper.
 test_files:
 - test/active_record_test.rb
 - test/cleaners_test.rb
+- test/data_cleansing_test.rb
 - test/ruby_test.rb
 - test/test_db.sqlite3
 - test/test_helper.rb