data-anonymization 0.1.2 → 0.2.0

data/blacklist_dsl.rb

@@ -2,8 +2,6 @@ system "bundle exec ruby whitelist_dsl.rb"
 
 require 'data-anonymization'
 
-FS = DataAnon::Strategy::Field
-
 DataAnon::Utils::Logging.logger.level = Logger::INFO
 
 database 'Chinook' do
@@ -12,7 +10,7 @@ database 'Chinook' do
 
   table 'MediaType' do
     primary_key 'MediaTypeId'
-    anonymize('Name').using FS::StringTemplate.new('Media Type 100#{row_number}')
+    anonymize('Name').using FieldStrategy::StringTemplate.new('Media Type 100#{row_number}')
   end
 
 end

data/data-anonymization.gemspec

@@ -18,5 +18,9 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_dependency('activerecord', '~> 3.2.8')
+  gem.add_dependency('composite_primary_keys', '~> 5.0.8')
   gem.add_dependency('activesupport', '~> 3.2.8')
+  gem.add_dependency('rgeo', '~> 0.3.15')
+  gem.add_dependency('rgeo-geojson', '~> 0.2.3')
+  gem.add_dependency('powerbar', '~> 1.0.8')
 end

data/lib/core/dsl.rb

@@ -4,7 +4,7 @@ module DataAnon
       include Utils::Logging
 
       def database(name, &block)
-        logger.debug "#{name} : Database"
+        logger.debug "Processing Database: #{name}"
         DataAnon::Core::Database.new(name).instance_eval &block
       end
 

data/lib/data-anonymization.rb

@@ -2,7 +2,10 @@ require "version"
 
 require "utils/logging"
 require "utils/random_int"
+require "utils/random_float"
 require "utils/random_string"
+require "utils/geojson_parser"
+require "utils/progress_bar"
 require "utils/resource"
 require "core/database"
 require "core/field"

data/lib/strategy/base.rb

@@ -1,3 +1,5 @@
+require 'powerbar'
+
 module DataAnon
   module Strategy
     class Base
@@ -14,10 +16,15 @@ module DataAnon
         self
       end
 
-      def primary_key field
-        @primary_key = field
+      def primary_key *fields
+        @primary_keys = fields
+      end
+
+      def is_primary_key? field
+        @primary_keys.select { |key| field.downcase == key.downcase }.length > 0
       end
 
+
       def whitelist *fields
         fields.each { |f| @fields[f.downcase] = DataAnon::Strategy::Field::Whitelist.new }
       end
@@ -43,23 +50,26 @@ module DataAnon
       end
 
       def dest_table
-        @dest_table ||= Utils::DestinationTable.create @name, @primary_key
+        @dest_table ||= Utils::DestinationTable.create @name, @primary_keys
       end
 
       def source_table
-        @source_table ||= Utils::SourceTable.create @name, @primary_key
+        @source_table ||= Utils::SourceTable.create @name, @primary_keys
       end
 
       def process
         logger.debug "Processing table #{@name} with fields strategies #{@fields}"
-        progress_logger.info "Table: #{@name} (#{source_table.count} records) "
-        index = 1
-        source_table.find_each(:batch_size => 100) do |record|
-          progress_logger.info "."
-          process_record index, record
-          index += 1
+        total = source_table.count
+        if total > 0
+          index = 1
+          progress_bar = DataAnon::Utils::ProgressBar.new @name, total
+          source_table.all.each do |record|
+            process_record index, record
+            index += 1
+            progress_bar.show(index)
+          end
+          progress_bar.close
         end
-        progress_logger.info " DONE\n"
       end
 
     end

data/lib/strategy/blacklist.rb

@@ -6,13 +6,14 @@ module DataAnon
         @fields.each do |field, strategy|
           database_field_name = record.attributes.select { |k,v| k.downcase == field }.keys[0]
           field_value = record.attributes[database_field_name]
-          unless field_value.nil? || database_field_name.downcase == @primary_key.downcase
+          unless field_value.nil? || is_primary_key?(database_field_name)
             field = DataAnon::Core::Field.new(database_field_name, field_value, index, record)
             record[database_field_name] = strategy.anonymize(field)
           end
         end
         record.save!
       end
+
     end
   end
 end

data/lib/strategy/field/contact/geojson_base.rb

@@ -0,0 +1,24 @@
+module DataAnon
+  module Strategy
+    module Field
+      class GeojsonBase
+
+        def self.region_US
+          self.new DataAnon::Utils::Resource.file('US_addresses.geojson')
+        end
+
+        def self.region_UK
+          self.new DataAnon::Utils::Resource.file('UK_addresses.geojson')
+        end
+
+        def initialize file_path
+          raise "Load and set the @values member variable in constructor"
+        end
+
+        def anonymize field
+          @values.sample
+        end
+      end
+    end
+  end
+end

data/lib/strategy/field/contact/random_address.rb

@@ -0,0 +1,17 @@
+module DataAnon
+  module Strategy
+    module Field
+
+
+      class RandomAddress  < GeojsonBase
+
+        def initialize file_path
+          @values = DataAnon::Utils::GeojsonParser.address(file_path)
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/strategy/field/contact/random_city.rb

@@ -0,0 +1,17 @@
+module DataAnon
+  module Strategy
+    module Field
+
+
+      class RandomCity < GeojsonBase
+
+        def initialize file_path
+          @values = DataAnon::Utils::GeojsonParser.city(file_path)
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/strategy/field/contact/random_phone_number.rb

@@ -0,0 +1,13 @@
+module DataAnon
+  module Strategy
+    module Field
+
+
+      class RandomPhoneNumber < FormattedStringNumber
+
+      end
+
+
+    end
+  end
+end

data/lib/strategy/field/contact/random_province.rb

@@ -0,0 +1,17 @@
+module DataAnon
+  module Strategy
+    module Field
+
+
+      class RandomProvince < GeojsonBase
+
+        def initialize file_path
+          @values = DataAnon::Utils::GeojsonParser.province(file_path)
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/strategy/field/contact/random_zipcode.rb

@@ -0,0 +1,17 @@
+module DataAnon
+  module Strategy
+    module Field
+
+
+      class RandomZipcode < GeojsonBase
+
+        def initialize file_path
+          @values = DataAnon::Utils::GeojsonParser.zipcode(file_path)
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/strategy/field/datetime/anonymize_date.rb

@@ -0,0 +1,39 @@
+module DataAnon
+  module Strategy
+    module Field
+
+      class AnonymizeDate
+
+
+        def self.only_month
+          self.new true, false
+        end
+
+        def self.only_day
+          self.new false, true
+        end
+
+        def initialize anonymize_month, anonymize_day
+
+          @anonymize_month = anonymize_month
+          @anonymize_day = anonymize_day
+
+        end
+
+        def anonymize field
+
+          original_time = field.value
+
+          year = original_time.year
+          month = @anonymize_month? DataAnon::Utils::RandomInt.generate(1,12) : original_time.month
+          days_in_month = Time.new(year,month,1,1,1,1).end_of_month.day
+          day = @anonymize_day? DataAnon::Utils::RandomInt.generate(1,days_in_month) : original_time.day
+
+          Date.new(year, month, day)
+        end
+
+
+      end
+    end
+  end
+end

data/lib/strategy/field/datetime/anonymize_datetime.rb

@@ -0,0 +1,15 @@
+module DataAnon
+  module Strategy
+    module Field
+
+      class AnonymizeDateTime < AnonymizeTime
+
+        private
+        def create_object(year, month, day, hour, min, sec)
+          DateTime.new(year, month, day, hour, min, sec)
+        end
+
+      end
+    end
+  end
+end

data/lib/strategy/field/datetime/anonymize_time.rb

@@ -0,0 +1,58 @@
+module DataAnon
+  module Strategy
+    module Field
+
+      class AnonymizeTime
+
+        DEFAULT_ANONYMIZATION = true
+
+        def self.only_month
+          self.new true, false, false, false
+        end
+
+        def self.only_day
+          self.new false, true, false, false
+        end
+
+        def self.only_hour
+          self.new false, false, true, false
+        end
+
+        def self.only_minute
+          self.new false, false, false, true
+        end
+
+        def initialize anonymize_month, anonymize_day, anonymize_hour, anonymize_min
+
+          @anonymize_month = anonymize_month
+          @anonymize_day = anonymize_day
+          @anonymize_hour = anonymize_hour
+          @anonymize_min = anonymize_min
+
+        end
+
+        def anonymize field
+
+          original_time = field.value
+
+          year = original_time.year
+          month = @anonymize_month? DataAnon::Utils::RandomInt.generate(1,12) : original_time.month
+          days_in_month = Time.new(year,month,1,1,1,1).end_of_month.day
+          day = @anonymize_day? DataAnon::Utils::RandomInt.generate(1,days_in_month) : original_time.day
+          hour = @anonymize_hour? DataAnon::Utils::RandomInt.generate(1,24) : original_time.hour
+          min = @anonymize_min? DataAnon::Utils::RandomInt.generate(1,60) : original_time.min
+          sec = original_time.sec
+
+          create_object(year, month, day, hour, min, sec)
+        end
+
+        private
+
+        def create_object(year, month, day, hour, min, sec)
+          Time.new(year, month, day, hour, min, sec)
+        end
+
+      end
+    end
+  end
+end

data/lib/strategy/field/datetime/date_delta.rb

@@ -0,0 +1,21 @@
+module DataAnon
+  module Strategy
+    module Field
+
+      class DateDelta
+
+        DEFAULT_DAY_DELTA = 10
+
+        def initialize day_delta = DEFAULT_DAY_DELTA
+          @day_delta = day_delta
+        end
+
+        def anonymize field
+          day_adjustment = DataAnon::Utils::RandomInt.generate(-@day_delta,@day_delta)
+          return field.value + day_adjustment.days
+        end
+
+      end
+    end
+  end
+end

data/lib/strategy/field/{date_time_delta.rb → datetime/date_time_delta.rb}

@@ -13,9 +13,9 @@ module DataAnon
         end
 
         def anonymize field
-          day_adjustment = @day_delta==0? 0 : (DataAnon::Utils::RandomInt.generate(-@day_delta,@day_delta))
-          minute_adjustment = @minute_delta==0? 0 :  (DataAnon::Utils::RandomInt.generate(-@minute_delta,@minute_delta))
-          return field.value + day_adjustment.days + minute_adjustment.minutes
+          day_adjustment = DataAnon::Utils::RandomInt.generate(-@day_delta,@day_delta)
+          minute_adjustment = DataAnon::Utils::RandomInt.generate(-@minute_delta,@minute_delta)
+          return field.value + (day_adjustment.days + minute_adjustment.minutes)
         end
 
       end

data/lib/strategy/field/datetime/time_delta.rb

@@ -0,0 +1,12 @@
+module DataAnon
+  module Strategy
+    module Field
+
+
+      class TimeDelta < DateTimeDelta
+      end
+
+
+    end
+  end
+end

data/lib/strategy/field/default_anon.rb

@@ -4,19 +4,24 @@ module DataAnon
 
       class DefaultAnon
 
-        FS = DataAnon::Strategy::Field
-        DEFAULT_STRATEGIES = {:string => FS::LoremIpsum.new,
-                              :integer => FS::RandomInt.new(18,70),
-                              :datetime => FS::DateTimeDelta.new,
-                              :boolean => FS::RandomBoolean.new
+        DEFAULT_STRATEGIES = {:string => FieldStrategy::LoremIpsum.new,
+                              :fixnum => FieldStrategy::RandomIntegerDelta.new(5),
+                              :bignum => FieldStrategy::RandomIntegerDelta.new(5000),
+                              :float => FieldStrategy::RandomFloatDelta.new(5.0),
+                              :datetime => FieldStrategy::DateTimeDelta.new,
+                              :time => FieldStrategy::TimeDelta.new,
+                              :date => FieldStrategy::DateDelta.new,
+                              :trueclass => FieldStrategy::RandomBoolean.new,
+                              :falseclass => FieldStrategy::RandomBoolean.new
         }
 
-        def initialize user_defaults
+        def initialize user_defaults = {}
           @user_defaults = DEFAULT_STRATEGIES.merge user_defaults
         end
 
         def anonymize field
-          strategy = @user_defaults[field.value.class.to_s.downcase.to_sym] || FS::Whitelist.new
+          strategy = @user_defaults[field.value.class.to_s.downcase.to_sym]
+          raise "No strategy defined for datatype #{field.value.class}" unless strategy
           strategy.anonymize field
         end