data_cleansing 0.9.0 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 8ba846025b7441eb5a93230b7fbd8ebe2a4d88e3
-  data.tar.gz: 4e209fd6ef57540a8b549d06c314ae4caeddbf59
+SHA256:
+  metadata.gz: 04a8c565168dcba73088d9e17afcda3ab49097a41fb55bcf880f2cd04e6d38bf
+  data.tar.gz: da24439e8431d81c19241b7ed579bd73170cfc8e61c1781bc103befa1e37d6fe
 SHA512:
-  metadata.gz: 7b464ca76d4c40f4621d86a32cd76bd4bc3e71e8b5eed18ac094ae651a8f0be58772a503fa096c6798b081cf3030363973b0d96cfd2cf45d6497e14a5b2717f1
-  data.tar.gz: e6933049c6200cadb6e398e3d2af8bae641534942a201c6ed7b8a47fc991f7a843d7b2d1b6cbc1c00f14d837f2de887ac6011f23c187fe33be3c6199a1e18cdf
+  metadata.gz: 28d9b9af8b18271bcbedd3930b0c1882ee162470959992ab2140f4a733377aaffadb0efef52d66ffeb86a93b1c10cdbcc78d61382d09fd85f8135596bd845628
+  data.tar.gz: f49c9062ffa4620c723cdb1d114f3baf649b63cb273dc67cadb34212cdf1d42c54a1ad4be90cff38e02f565c2135d18c3d58ccd0e90c3031e65dfe3fbef0e01c

data/README.md

@@ -1,7 +1,7 @@
 data_cleansing
 ==============
 
-Data Cleansing framework for Ruby, Rails, Mongoid and MongoMapper.
+Data Cleansing framework for Ruby.
 
 * http://github.com/reidmorrison/data_cleansing
 
@@ -12,12 +12,8 @@ or trailing blanks and even newlines.
 Similarly it would be useful to be able to attach a cleansing solution to a field
 in a model and have the data cleansed transparently when required.
 
-DataCleansing is a framework that allows any data cleansing to be applied to
-specific attributes or fields. At this time it does not supply the cleaning
-solutions themselves since they are usually straight forward, or so complex
-that they don't tend to be too useful to others. However, over time built-in
-cleansing solutions may be added. Feel free to submit any suggestions via a ticket
-or pull request.
+DataCleansing is a framework that allows data cleansing to be applied to
+specific attributes or fields.
 
 ## Features
 
@@ -297,24 +293,6 @@ Install the Gem with bundler
 
     bundle install
 
-## Architecture
-
-DataCleansing has been designed to support externalized data cleansing routines.
-In this way the data cleansing routine itself can be loaded from a datastore and
-applied dynamically at runtime.
-Although not supported out of the box, this design allows for example for the
-data cleansing routines to be stored in something like [ZooKeeper](http://zookeeper.apache.org/).
-Then any changes to the data cleansing routines can be pushed out immediately to
-every server that needs it.
-
-DataCleansing is designed to support any Ruby model. In this way it can be used
-in just about any ORM or DOM. For example, it currently easily supports both
-Rails and Mongoid models. Some extensions have been added to support these frameworks.
-
-For example, in Rails it obtains the raw data value before Rails has converted it.
-Which is useful for cleansing integer or float fields as raw strings before Rails
-tries to convert it to an integer or float.
-
 ## Dependencies
 
 DataCleansing requires the following dependencies

data/Rakefile

@@ -1,27 +1,31 @@
-require 'rake/clean'
-require 'rake/testtask'
+# Setup bundler to avoid having to run bundle exec all the time.
+require "rubygems"
+require "bundler/setup"
 
-require_relative 'lib/data_cleansing/version'
+require "rake/testtask"
+require_relative "lib/data_cleansing/version"
 
 task :gem do
-  system 'gem build data_cleansing.gemspec'
+  system "gem build data_cleansing.gemspec"
 end
 
 task publish: :gem do
   system "git tag -a v#{DataCleansing::VERSION} -m 'Tagging #{DataCleansing::VERSION}'"
-  system 'git push --tags'
+  system "git push --tags"
   system "gem push data_cleansing-#{DataCleansing::VERSION}.gem"
   system "rm data_cleansing-#{DataCleansing::VERSION}.gem"
 end
 
-desc 'Run Test Suite'
-task :test do
-  Rake::TestTask.new(:functional) do |t|
-    t.test_files = FileList['test/**/*_test.rb']
-    t.verbose    = true
-  end
-
-  Rake::Task['functional'].invoke
+Rake::TestTask.new(:test) do |t|
+  t.pattern = "test/**/*_test.rb"
+  t.verbose = true
+  t.warning = false
 end
 
-task default: :test
+# By default run tests against all appraisals
+if !ENV["APPRAISAL_INITIALIZED"] && !ENV["TRAVIS"]
+  require "appraisal"
+  task default: :appraisal
+else
+  task default: :test
+end

data/lib/data_cleansing/cleaners.rb

@@ -1,4 +1,4 @@
-require 'uri'
+require "cgi"
 module Cleaners
   # Strip leading and trailing whitespace
   module Strip
@@ -20,6 +20,16 @@ module Cleaners
   end
   DataCleansing.register_cleaner(:upcase, Upcase)
 
+  # Convert to downcase
+  module Downcase
+    def self.call(string)
+      return string unless string.is_a?(String)
+
+      string.downcase! || string
+    end
+  end
+  DataCleansing.register_cleaner(:downcase, Downcase)
+
   # Remove all non-word characters, including whitespace
   module RemoveNonWord
     NOT_WORDS = Regexp.compile(/\W/)
@@ -27,7 +37,7 @@ module Cleaners
     def self.call(string)
       return string unless string.is_a?(String)
 
-      string.gsub!(NOT_WORDS, '') || string
+      string.gsub!(NOT_WORDS, "") || string
     end
   end
   DataCleansing.register_cleaner(:remove_non_word, RemoveNonWord)
@@ -39,12 +49,16 @@ module Cleaners
     def self.call(string)
       return string unless string.is_a?(String)
 
+      # Strip invalid characters, since they are non printable
+      unless string.valid_encoding?
+        string = string.encode(string.encoding, invalid: :replace, undef: :replace, replace: "")
+      end
       string.gsub!(NOT_PRINTABLE, '') || string
     end
   end
   DataCleansing.register_cleaner(:remove_non_printable, RemoveNonPrintable)
 
-  # Remove HTML Markup
+  # Unescape HTML Markup ( case-insensitive )
   module ReplaceHTMLMarkup
     HTML_MARKUP = Regexp.compile(/&(amp|quot|gt|lt|apos|nbsp);/in)
 
@@ -53,18 +67,18 @@ module Cleaners
 
       string.gsub!(HTML_MARKUP) do |match|
         case match.downcase
-        when '&' then
-          '&'
-        when '"' then
+        when "&"
+          "&"
+        when """
           '"'
-        when '>' then
-          '>'
-        when '<' then
-          '<'
-        when '&apos;' then
+        when "&gt;"
+          ">"
+        when "&lt;"
+          "<"
+        when "&apos;"
           "'"
-        when '&nbsp;' then
-          ' '
+        when "&nbsp;"
+          " "
         else
           "&#{match};"
         end
@@ -77,7 +91,7 @@ module Cleaners
     def self.call(string)
       return string unless string.is_a?(String)
 
-      URI.unescape(string)
+      CGI.unescape(string)
     end
   end
   DataCleansing.register_cleaner(:unescape_uri, UnescapeURI)
@@ -86,7 +100,7 @@ module Cleaners
     def self.call(string)
       return string unless string.is_a?(String)
 
-      URI.escape(string)
+      CGI.escape(string)
     end
   end
   DataCleansing.register_cleaner(:escape_uri, EscapeURI)
@@ -98,7 +112,7 @@ module Cleaners
     def self.call(string)
       return string unless string.is_a?(String)
 
-      string.gsub!(WHITESPACE, ' ') || string
+      string.gsub!(WHITESPACE, " ") || string
     end
   end
   DataCleansing.register_cleaner(:compress_whitespace, CompressWhitespace)
@@ -111,7 +125,7 @@ module Cleaners
     def self.call(string)
       return string unless string.is_a?(String)
 
-      string.gsub!(DIGITS, '')
+      string.gsub!(DIGITS, "")
       string.length > 0 ? string : nil
     end
   end
@@ -120,13 +134,13 @@ module Cleaners
   # Returns [Integer] after removing all non-digit characters, except '.'
   # Returns nil if no digits are present in the string.
   module StringToInteger
-    NUMERIC = Regexp.compile(/[^0-9\.]/)
+    NUMERIC = Regexp.compile(/[^0-9.]/)
 
     def self.call(string)
       return string unless string.is_a?(String)
 
       # Remove Non-Digit Chars, except for '.'
-      string.gsub!(NUMERIC, '')
+      string.gsub!(NUMERIC, "")
       string.length > 0 ? string.to_i : nil
     end
   end
@@ -135,13 +149,13 @@ module Cleaners
   # Returns [Integer] after removing all non-digit characters, except '.'
   # Returns nil if no digits are present in the string.
   module StringToFloat
-    NUMERIC = Regexp.compile(/[^0-9\.]/)
+    NUMERIC = Regexp.compile(/[^0-9.]/)
 
     def self.call(string)
       return string unless string.is_a?(String)
 
       # Remove Non-Digit Chars, except for '.'
-      string.gsub!(NUMERIC, '')
+      string.gsub!(NUMERIC, "")
       string.length > 0 ? string.to_f : nil
     end
   end

data/lib/data_cleansing/cleanse.rb

@@ -1,4 +1,4 @@
-require 'data_cleansing/cleaners'
+require "data_cleansing/cleaners"
 module DataCleansing
   # Mix-in to add cleaner
   module Cleanse
@@ -7,10 +7,10 @@ module DataCleansing
     module ClassMethods
       # Define how to cleanse one or more attributes
       def cleanse(*args)
-        last = args.last
+        last       = args.last
         attributes = args.dup
-        params = (last.is_a?(Hash) && last.instance_of?(Hash)) ? attributes.pop.dup : {}
-        cleaners = Array(params.delete(:cleaner))
+        params     = last.is_a?(Hash) && last.instance_of?(Hash) ? attributes.pop.dup : {}
+        cleaners   = Array(params.delete(:cleaner))
         raise(ArgumentError, "Mandatory :cleaner parameter is missing: #{params.inspect}") unless cleaners
 
         cleaner = DataCleansingCleaner.new(cleaners, attributes, params)
@@ -34,6 +34,7 @@ module DataCleansing
       def after_cleanse(*methods)
         methods.each do |m|
           raise "Method #{m.inspect} must be a symbol" unless m.is_a?(Symbol)
+
           data_cleansing_after_cleaners << m unless data_cleansing_after_cleaners.include?(m)
         end
       end
@@ -53,12 +54,12 @@ module DataCleansing
       #
       # Warning: If any of the cleaners read or write to other object attributes
       #          then a valid object instance must be supplied
-      def cleanse_attribute(attribute_name, value, object=nil)
+      def cleanse_attribute(attribute_name, value, object = nil)
         return if value.nil?
 
         # Collect parent cleaners first, starting with the top parent
         cleaners = []
-        klass = self
+        klass    = self
         while klass != Object
           if klass.respond_to?(:data_cleansing_attribute_cleaners)
             cleaners += klass.data_cleansing_attribute_cleaners[:all] || []
@@ -66,8 +67,9 @@ module DataCleansing
           end
           klass = klass.superclass
         end
-        cleansed_value = value.dup
-        cleaners.reverse_each {|cleaner| cleansed_value = data_cleansing_clean(cleaner, cleansed_value, object) if cleaner}
+        # Support Integer values
+        cleansed_value = value.is_a?(Integer) ? value : value.dup
+        cleaners.reverse_each { |cleaner| cleansed_value = data_cleansing_clean(cleaner, cleansed_value, object) if cleaner }
         cleansed_value
       end
 
@@ -90,37 +92,23 @@ module DataCleansing
 
       # Returns the supplied value cleansed using the supplied cleaner
       # Parameters
-      #   object
+      #   binding
       #     If supplied the cleansing will be performed within the scope of
-      #     that object so that cleaners can read and write to attributes
-      #     of that object
+      #     that binding so that cleaners can read and write to attributes
+      #     of that binding
       #
       # No logging of cleansing is performed by this method since the value
       # itself is not modified
-      def data_cleansing_clean(cleaner_struct, value, object=nil)
+      def data_cleansing_clean(cleaner_struct, value, binding = nil)
         return if cleaner_struct.nil? || value.nil?
+
         # Duplicate value in case cleaner uses methods such as gsub!
         new_value = value.is_a?(String) ? value.dup : value
         cleaner_struct.cleaners.each do |name|
-          # Cleaner itself could be a custom Proc, otherwise do a global lookup for it
-          proc = name.is_a?(Proc) ? name : DataCleansing.cleaner(name.to_sym)
-          raise "No cleaner defined for #{name.inspect}" unless proc
-
-          if proc.is_a?(Proc)
-            new_value = if object
-              # Call the cleaner proc within the scope (binding) of the object
-              proc.arity == 1 ? object.instance_exec(new_value, &proc) : object.instance_exec(new_value, cleaner_struct.params, &proc)
-            else
-              proc.arity == 1 ? proc.call(new_value) : proc.call(new_value, cleaner_struct.params)
-            end
-          else
-            new_value = (proc.method(:call).arity == 1 ? proc.call(new_value) : proc.call(new_value, cleaner_struct.params))
-          end
-
+          new_value = DataCleansing.clean(name, new_value, cleaner_struct.params, binding)
         end
         new_value
       end
-
     end
 
     module InstanceMethods
@@ -131,23 +119,23 @@ module DataCleansing
       #
       # Note: At this time the changes returned does not include any fields
       #       modified in any of the after_cleaner methods
-      def cleanse_attributes!(verbose=DataCleansing.logger.debug?)
+      def cleanse_attributes!(verbose = DataCleansing.logger.debug?)
         changes = {}
-        DataCleansing.logger.benchmark_info("#{self.class.name}#cleanse_attributes!", :payload => changes) do
+        DataCleansing.logger.benchmark_info("#{self.class.name}#cleanse_attributes!", payload: changes) do
           # Collect parent cleaners first, starting with the top parent
-          cleaners = [self.class.send(:data_cleansing_cleaners)]
+          cleaners       = [self.class.send(:data_cleansing_cleaners)]
           after_cleaners = [self.class.send(:data_cleansing_after_cleaners)]
-          klass = self.class.superclass
+          klass          = self.class.superclass
           while klass != Object
             cleaners << klass.send(:data_cleansing_cleaners) if klass.respond_to?(:data_cleansing_cleaners)
             after_cleaners << klass.send(:data_cleansing_after_cleaners) if klass.respond_to?(:data_cleansing_after_cleaners)
             klass = klass.superclass
           end
           # Capture all modified fields if log_level is :debug or :trace
-          cleaners.reverse_each {|cleaner| changes.merge!(data_cleansing_execute_cleaners(cleaner, verbose))}
+          cleaners.reverse_each { |cleaner| changes.merge!(data_cleansing_execute_cleaners(cleaner, verbose)) }
 
           # Execute the after cleaners, starting with the parent after cleanse methods
-          after_cleaners.reverse_each {|a| a.each {|method| send(method)} }
+          after_cleaners.reverse_each { |a| a.each { |method| send(method) } }
         end
         changes
       end
@@ -176,15 +164,9 @@ module DataCleansing
           # Special case to include :all fields
           # Only works with ActiveRecord based models, not supported with regular Ruby models
           if attrs.include?(:all) && defined?(ActiveRecord) && respond_to?(:attributes)
-            attrs = attributes.keys.collect{|i| i.to_sym}
+            attrs = attributes.keys.collect { |i| i.to_sym }
             attrs.delete(:id)
 
-            # Remove serialized_attributes if any, from the :all condition
-            if self.class.respond_to?(:serialized_attributes)
-              serialized_attrs = self.class.serialized_attributes.keys
-              attrs -= serialized_attrs.collect{|i| i.to_sym} if serialized_attrs
-            end
-
             # Replace any encrypted attributes with their non-encrypted versions if any
             if defined?(SymmetricEncryption) && self.class.respond_to?(:encrypted_attributes)
               self.class.encrypted_attributes.each_pair do |clear, encrypted|
@@ -205,51 +187,48 @@ module DataCleansing
           attrs.each do |attr|
             # Under ActiveModel for Rails and Mongoid need to retrieve raw value
             # before data type conversion
-            value = if respond_to?(:read_attribute_before_type_cast) && has_attribute?(attr.to_s)
-              read_attribute_before_type_cast(attr.to_s)
-            else
-              send(attr.to_sym)
-            end
+            value =
+              if respond_to?(:read_attribute_before_type_cast) && has_attribute?(attr.to_s)
+                read_attribute_before_type_cast(attr.to_s)
+              else
+                send(attr.to_sym)
+              end
 
             # No need to clean if attribute is nil
-            unless value.nil?
-              new_value = self.class.send(:data_cleansing_clean,cleaner_struct, value, self)
-
-              if new_value != value
-                # Update value only if it has changed
-                send("#{attr.to_sym}=".to_sym, new_value)
-
-                # Capture changed attributes
-                if changes
-                  # Mask sensitive attributes when logging
-                  masked = DataCleansing.masked_attributes.include?(attr.to_sym)
-                  new_value = :masked if masked && !new_value.nil?
-                  if previous = changes[attr.to_sym]
-                    previous[:after] = new_value
-                  else
-                    if new_value.nil? || verbose
-                      changes[attr.to_sym] = {
-                        :before => masked ? :masked : value,
-                        :after  => new_value
-                      }
-                    end
-                  end
-                end
-              end
+            next if value.nil?
+
+            new_value = self.class.send(:data_cleansing_clean, cleaner_struct, value, self)
+
+            next unless new_value != value
+
+            # Update value only if it has changed
+            send("#{attr.to_sym}=".to_sym, new_value)
+
+            # Capture changed attributes
+            next unless changes
+
+            # Mask sensitive attributes when logging
+            masked    = DataCleansing.masked_attributes.include?(attr.to_sym)
+            new_value = :masked if masked && !new_value.nil?
+            if previous = changes[attr.to_sym]
+              previous[:after] = new_value
+            elsif new_value.nil? || verbose
+              changes[attr.to_sym] = {
+                before: masked ? :masked : value,
+                after:  new_value
+              }
             end
           end
         end
         changes
       end
-
     end
 
     def self.included(base)
       base.class_eval do
-        extend  DataCleansing::Cleanse::ClassMethods
+        extend DataCleansing::Cleanse::ClassMethods
         include DataCleansing::Cleanse::InstanceMethods
       end
     end
   end
-
 end

data/lib/data_cleansing/data_cleansing.rb

@@ -9,6 +9,7 @@ module DataCleansing
   # Replaces any existing cleaner with the same name
   def self.register_cleaner(name, cleaner = nil, &block)
     raise "Must supply a Proc with the cleaner" unless block || cleaner
+
     @@global_cleaners[name.to_sym] = cleaner || block
   end
 
@@ -19,7 +20,7 @@ module DataCleansing
 
   # Register Attributes to be masked out in any log output
   def self.register_masked_attributes(*attributes)
-    attributes.each {|attr| @@masked_attributes << attr.to_sym }
+    attributes.each { |attr| @@masked_attributes << attr.to_sym }
   end
 
   # Returns the Global list of attributes to mask in any log output
@@ -27,4 +28,21 @@ module DataCleansing
     @@masked_attributes.freeze
   end
 
+  # Run the specified cleanser against the supplied value
+  def self.clean(name, value, params = nil, binding = nil)
+    # Cleaner itself could be a custom Proc, otherwise do a global lookup for it
+    proc = name.is_a?(Proc) ? name : DataCleansing.cleaner(name.to_sym)
+    raise(ArgumentError, "No cleaner defined for #{name.inspect}") unless proc
+
+    if proc.is_a?(Proc)
+      if binding
+        # Call the cleaner proc within the scope (binding) of the binding
+        proc.arity == 1 ? binding.instance_exec(value, &proc) : binding.instance_exec(value, params, &proc)
+      else
+        proc.arity == 1 ? proc.call(value) : proc.call(value, params)
+      end
+    else
+      (proc.method(:call).arity == 1 ? proc.call(value) : proc.call(value, params))
+    end
+  end
 end

data/lib/data_cleansing/railtie.rb

@@ -1,6 +1,5 @@
 module RubySkynet #:nodoc:
   class Railtie < Rails::Railtie #:nodoc:
-
     # Exposes DataCleansing configuration to the Rails application configuration.
     #
     # @example Set up configuration in the Rails app.

data/lib/data_cleansing/version.rb

@@ -1,3 +1,3 @@
 module DataCleansing
-  VERSION = '0.9.0'
+  VERSION = "1.0.3"
 end

data/lib/data_cleansing.rb

@@ -1,13 +1,11 @@
-require 'concurrent'
-require 'semantic_logger'
-require 'data_cleansing/version'
-require 'data_cleansing/data_cleansing'
+require "concurrent"
+require "semantic_logger"
+require "data_cleansing/version"
+require "data_cleansing/data_cleansing"
 
 module DataCleansing
-  autoload :Cleanse, 'data_cleansing/cleanse'
+  autoload :Cleanse, "data_cleansing/cleanse"
 end
 
 # Rails Extensions
-if defined?(Rails)
-  require 'data_cleansing/railtie'
-end
+require "data_cleansing/railtie" if defined?(Rails)