data-anonymization 0.1.2

data/lib/core/dsl.rb

@@ -0,0 +1,16 @@
+module DataAnon
+  module Core
+    module DSL
+      include Utils::Logging
+
+      def database(name, &block)
+        logger.debug "#{name} : Database"
+        DataAnon::Core::Database.new(name).instance_eval &block
+      end
+
+    end
+  end
+end
+
+include DataAnon::Core::DSL
+

data/lib/core/field.rb

@@ -0,0 +1,18 @@
+module DataAnon
+  module Core
+
+    class Field
+
+      def initialize name, value, row_number, ar_record
+        @name = name
+        @value = value
+        @row_number = row_number
+        @ar_record = ar_record
+      end
+
+      attr_accessor :name, :value, :row_number, :ar_record
+
+    end
+
+  end
+end

data/lib/data-anonymization.rb

@@ -0,0 +1,12 @@
+require "version"
+
+require "utils/logging"
+require "utils/random_int"
+require "utils/random_string"
+require "utils/resource"
+require "core/database"
+require "core/field"
+require "strategy/strategies"
+require "utils/database"
+require "core/dsl"
+

data/lib/strategy/base.rb

@@ -0,0 +1,67 @@
+module DataAnon
+  module Strategy
+    class Base
+      include Utils::Logging
+
+      def initialize name, user_strategies
+        @name = name
+        @user_strategies = user_strategies
+        @fields = {}
+      end
+
+      def process_fields &block
+        self.instance_eval &block
+        self
+      end
+
+      def primary_key field
+        @primary_key = field
+      end
+
+      def whitelist *fields
+        fields.each { |f| @fields[f.downcase] = DataAnon::Strategy::Field::Whitelist.new }
+      end
+
+      def fields
+        @fields
+      end
+
+      def anonymize *fields, &block
+        if block.nil?
+          fields.each { |f| @fields[f.downcase] = DataAnon::Strategy::Field::DefaultAnon.new(@user_strategies) }
+          temp = self
+          return Class.new do
+            @temp_fields = fields
+            @table_fields = temp.fields
+            def self.using field_strategy
+              @temp_fields.each { |f| @table_fields[f.downcase] = field_strategy }
+            end
+          end
+        else
+          fields.each { |f| @fields[f.downcase] = DataAnon::Strategy::Field::Anonymous.new(&block) }
+        end
+      end
+
+      def dest_table
+        @dest_table ||= Utils::DestinationTable.create @name, @primary_key
+      end
+
+      def source_table
+        @source_table ||= Utils::SourceTable.create @name, @primary_key
+      end
+
+      def process
+        logger.debug "Processing table #{@name} with fields strategies #{@fields}"
+        progress_logger.info "Table: #{@name} (#{source_table.count} records) "
+        index = 1
+        source_table.find_each(:batch_size => 100) do |record|
+          progress_logger.info "."
+          process_record index, record
+          index += 1
+        end
+        progress_logger.info " DONE\n"
+      end
+
+    end
+  end
+end

data/lib/strategy/blacklist.rb

@@ -0,0 +1,18 @@
+module DataAnon
+  module Strategy
+    class Blacklist < DataAnon::Strategy::Base
+
+      def process_record index, record
+        @fields.each do |field, strategy|
+          database_field_name = record.attributes.select { |k,v| k.downcase == field }.keys[0]
+          field_value = record.attributes[database_field_name]
+          unless field_value.nil? || database_field_name.downcase == @primary_key.downcase
+            field = DataAnon::Core::Field.new(database_field_name, field_value, index, record)
+            record[database_field_name] = strategy.anonymize(field)
+          end
+        end
+        record.save!
+      end
+    end
+  end
+end

data/lib/strategy/field/anonymize_time.rb

@@ -0,0 +1,57 @@
+module DataAnon
+  module Strategy
+    module Field
+
+      class AnonymizeTime
+
+        DEFAULT_ANONYMIZATION = true
+
+        def self.only_month
+          self.new true, false, false, false, false
+        end
+
+        def self.only_day
+          self.new false, true, false, false, false
+        end
+
+        def self.only_hour
+          self.new false, false, true, false, false
+        end
+
+        def self.only_minute
+          self.new false, false, false, true, false
+        end
+
+        def initialize anonymize_month = DEFAULT_ANONYMIZATION, anonymize_day = DEFAULT_ANONYMIZATION, anonymize_hour = DEFAULT_ANONYMIZATION, anonymize_min = DEFAULT_ANONYMIZATION, anonymize_sec = DEFAULT_ANONYMIZATION
+
+          @anonymize_month = anonymize_month
+          @anonymize_day = anonymize_day
+          @anonymize_hour = anonymize_hour
+          @anonymize_min = anonymize_min
+          @anonymize_sec = anonymize_sec
+
+        end
+
+        def anonymize field
+
+          provided_time = field.value
+          year = provided_time.year
+          month = @anonymize_month? DataAnon::Utils::RandomInt.generate(1,12) : provided_time.month
+          day = @anonymize_day? DataAnon::Utils::RandomInt.generate(1,31) : provided_time.day
+          hour = @anonymize_hour? DataAnon::Utils::RandomInt.generate(1,24) : provided_time.hour
+          min = @anonymize_min? DataAnon::Utils::RandomInt.generate(1,60) : provided_time.min
+          sec = @anonymize_sec? DataAnon::Utils::RandomInt.generate(1,60) : provided_time.sec
+
+          create_object(day, hour, min, month, sec, year)
+        end
+
+        private
+
+        def create_object(day, hour, min, month, sec, year)
+          Time.new(year, month, day, hour, min, sec)
+        end
+
+      end
+    end
+  end
+end

data/lib/strategy/field/anonymous.rb

@@ -0,0 +1,21 @@
+module DataAnon
+  module Strategy
+    module Field
+
+
+      class Anonymous
+
+        def initialize &block
+          @block = block
+        end
+
+        def anonymize field
+          @block.call field
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/strategy/field/date_time_delta.rb

@@ -0,0 +1,24 @@
+module DataAnon
+  module Strategy
+    module Field
+
+      class DateTimeDelta
+
+        DEFAULT_DAY_DELTA = 10
+        DEFAULT_MINUTE_DELTA = 30
+
+        def initialize day_delta = DEFAULT_DAY_DELTA, minute_delta = DEFAULT_MINUTE_DELTA
+          @day_delta = day_delta
+          @minute_delta =  minute_delta
+        end
+
+        def anonymize field
+          day_adjustment = @day_delta==0? 0 : (DataAnon::Utils::RandomInt.generate(-@day_delta,@day_delta))
+          minute_adjustment = @minute_delta==0? 0 :  (DataAnon::Utils::RandomInt.generate(-@minute_delta,@minute_delta))
+          return field.value + day_adjustment.days + minute_adjustment.minutes
+        end
+
+      end
+    end
+  end
+end

data/lib/strategy/field/default_anon.rb

@@ -0,0 +1,28 @@
+module DataAnon
+  module Strategy
+    module Field
+
+      class DefaultAnon
+
+        FS = DataAnon::Strategy::Field
+        DEFAULT_STRATEGIES = {:string => FS::LoremIpsum.new,
+                              :integer => FS::RandomInt.new(18,70),
+                              :datetime => FS::DateTimeDelta.new,
+                              :boolean => FS::RandomBoolean.new
+        }
+
+        def initialize user_defaults
+          @user_defaults = DEFAULT_STRATEGIES.merge user_defaults
+        end
+
+        def anonymize field
+          strategy = @user_defaults[field.value.class.to_s.downcase.to_sym] || FS::Whitelist.new
+          strategy.anonymize field
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/strategy/field/distinct_column_values.rb

@@ -0,0 +1,25 @@
+module DataAnon
+  module Strategy
+    module Field
+
+      class DistinctColumnValues
+        include Utils::Logging
+
+        def initialize table_name, field_name
+          source = Utils::SourceTable.create table_name
+          @values = source.select(field_name).uniq.collect { |record| record[field_name]}
+          logger.debug "For field strategy #{table_name}:#{field_name} using values #{@values} "
+
+        end
+
+        def anonymize field
+          return @values[0] if @values.length == 1
+          @values[DataAnon::Utils::RandomInt.generate(0,(@values.length - 1))]
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/strategy/field/fields.rb

@@ -0,0 +1,23 @@
+require 'strategy/field/whitelist'
+require 'strategy/field/string_template'
+require 'strategy/field/user_name_template'
+require 'strategy/field/random_string'
+require 'strategy/field/random_int'
+require 'strategy/field/random_boolean'
+require 'strategy/field/anonymize_time'
+require 'strategy/field/random_integer_delta'
+require 'strategy/field/random_float_delta'
+require 'strategy/field/random_selection'
+require 'strategy/field/distinct_column_values'
+require 'strategy/field/lorem_ipsum'
+require 'strategy/field/gmail_template'
+require 'strategy/field/date_time_delta'
+require 'strategy/field/default_anon'
+require 'strategy/field/random_email'
+require 'strategy/field/random_mailinator_email'
+require 'strategy/field/random_phone_number'
+require 'strategy/field/random_first_name'
+require 'strategy/field/random_last_name'
+require 'strategy/field/random_full_name'
+require 'strategy/field/random_user_name'
+require 'strategy/field/anonymous'

data/lib/strategy/field/gmail_template.rb

@@ -0,0 +1,17 @@
+module DataAnon
+  module Strategy
+    module Field
+      class GmailTemplate
+
+        def initialize gmail_address = nil
+          @gmail_address = gmail_address
+        end
+
+        def anonymize field
+          username = @gmail_address[0,@gmail_address.index('@')]
+          "#{username}+#{field.row_number}@gmail.com"
+        end
+      end
+    end
+  end
+end

data/lib/strategy/field/lorem_ipsum.rb

@@ -0,0 +1,29 @@
+module DataAnon
+  module Strategy
+    module Field
+
+
+      class LoremIpsum
+
+        DEFAULT_TEXT = <<-default
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed quis nulla quis ligula bibendum dignissim. Nullam elementum convallis mauris, at ultrices odio dignissim dapibus. Etiam vitae neque lorem, a luctus purus. In at diam mi, sit amet dapibus magna. Maecenas tincidunt tortor id dolor tristique dictum. Morbi pulvinar odio ut lorem gravida ac varius orci ultrices. Nulla id arcu dui, sit amet commodo augue. Curabitur elit elit, semper quis tincidunt at, auctor et tortor.
+Quisque ut enim arcu. Praesent orci mi, tincidunt non sodales a, blandit ac nunc. Phasellus sed erat a nibh suscipit molestie sed a augue. Aliquam pretium ultricies nibh. Sed sit amet accumsan sapien. Pellentesque urna orci, iaculis eu lacinia ac, consequat vel elit. Suspendisse aliquet tortor et urna varius non ullamcorper augue tempus. Phasellus pretium, nulla eu adipiscing viverra, purus est fermentum enim, ut fringilla ligula lectus quis est. Phasellus quis scelerisque ligula. Cras accumsan lobortis egestas. Ut quis orci sem, sed gravida orci.
+Vestibulum eget odio nisl, nec ornare ante. Aenean tristique, nisl eget lacinia aliquam, neque lectus lacinia enim, id ullamcorper nisl lorem vitae enim. Sed vulputate condimentum convallis. Ut viverra tincidunt arcu ac egestas. Quisque ut neque nec quam suscipit ornare a ornare est. Nulla facilisi. Mauris facilisis eleifend neque eget egestas. Vestibulum egestas dui eleifend urna pharetra a hendrerit quam sagittis. Duis ut turpis convallis diam interdum congue. In hac habitasse platea dictumst. Nulla a erat eget tortor tempor consectetur. Fusce euismod congue risus in feugiat. Sed rutrum vehicula lectus et vehicula. In porttitor malesuada sem at auctor.
+Maecenas lacinia placerat augue quis posuere. Cras eu augue quam, eu malesuada sem. Proin facilisis iaculis lectus, vel hendrerit nulla tristique quis. Donec risus mauris, vulputate tristique feugiat nec, imperdiet sed sapien. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Aenean vitae aliquam magna. Donec tempor, ipsum non dapibus elementum, est sem hendrerit nulla, scelerisque sollicitudin lacus mauris eu libero. Vivamus turpis justo, ullamcorper sed ullamcorper quis, tempor in elit. Sed nisl erat, laoreet at adipiscing quis, lobortis et est. Duis congue iaculis mollis. Curabitur ligula turpis, malesuada non feugiat vitae, ullamcorper non nibh. Aliquam adipiscing pellentesque leo nec molestie. Donec tempor eleifend libero, at rutrum velit semper a. Sed tincidunt dictum lorem eu egestas.
+Sed at iaculis risus. Nulla aliquet vulputate nulla, nec euismod sem porta quis. Aliquam erat volutpat. Sed tincidunt pharetra metus, in facilisis nunc suscipit ut. Nunc placerat vulputate sapien, elementum varius mi viverra eget. Nam hendrerit felis et arcu ultrices vehicula. Phasellus condimentum ornare orci sed placerat. Sed vel rutrum lorem. Fusce id bibendum ipsum.
+        default
+
+        def initialize text = nil
+          @text = text || DEFAULT_TEXT
+        end
+
+        def anonymize field
+          @text[0, field.value.length]
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/strategy/field/random_boolean.rb

@@ -0,0 +1,19 @@
+module DataAnon
+  module Strategy
+    module Field
+
+
+      class RandomBoolean
+
+        BOOL_VALUES = [true,false]
+
+        def anonymize field
+           BOOL_VALUES.sample
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/strategy/field/random_email.rb

@@ -0,0 +1,31 @@
+module DataAnon
+  module Strategy
+    module Field
+
+      class RandomEmail
+
+        TLDS = ['com','org','net','edu','gov','mil','biz','info']
+
+
+        def initialize hostname = nil, tld = nil
+          @hostname = hostname
+          @tld = tld
+        end
+
+        def anonymize field
+
+          username_length = DataAnon::Utils::RandomInt.generate(5,15)
+          host_name_length = DataAnon::Utils::RandomInt.generate(2,10)
+
+          username = DataAnon::Utils::RandomString.generate(username_length)
+          hostname = @hostname || DataAnon::Utils::RandomString.generate(host_name_length)
+          tld = @tld || TLDS[rand(TLDS.length)]
+
+          return username + "@" + hostname + "." + tld
+
+        end
+      end
+    end
+
+  end
+end

data/lib/strategy/field/random_first_name.rb

@@ -0,0 +1,18 @@
+module DataAnon
+  module Strategy
+    module Field
+
+      class RandomFirstName
+
+        def initialize file_path = nil
+          file = file_path || DataAnon::Utils::Resource.file('first_names.txt')
+          @names = File.read(file).split
+        end
+
+        def anonymize field
+          return @names[rand(@names.size)]
+        end
+      end
+    end
+  end
+end