data-anonymization 0.3.0 → 0.5.0

data/bin/datanon

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require 'thor'
+require 'data-anonymization'
+
+class AnonymizationCLI < Thor
+
+  include Thor::Actions
+
+  desc "generate_rdbms_dsl", "Generates a base anonymization script(whitelist strategy) for a RDBMS database using the database schema"
+
+  def generate_rdbms_dsl
+
+    configuration_hash = {:adapter => options["adapter"],
+                           :host => options["host"],
+                           :port => options["port"],
+                           :database => options["database"],
+                           :username => options["username"],
+                           :password => options["password"]
+                          }
+    create_file "rdbms_whitelist_generated.rb"
+    DataAnon::ThorHelpers::RDBMSDSLGenerator.new.generate_whitelist_script(configuration_hash)
+  end
+
+  method_option :adapter, :required => true, :aliases => "-a", :desc => "Activerecord database adapter to be used [required]", :for => :generate_rdbms_dsl
+  method_option :host, :required => true, :aliases => "-h", :desc => "Source Database host [required]", :for => :generate_rdbms_dsl
+  method_option :database, :required => true, :aliases => "-d", :desc => "Database name [required]", :for => :generate_rdbms_dsl
+  method_option :port, :aliases => "-p", :desc => "Port to connect to. If not provided default port provided by AR will be used", :for => :generate_rdbms_dsl
+  method_option :username, :aliases => "-u", :desc => "Username", :for => :generate_rdbms_dsl
+  method_option :password, :aliases => "-w", :desc => "Password", :for => :generate_rdbms_dsl
+
+  desc "generate_mongo_dsl", "Generates a base anonymization script(whitelist strategy) for a Mongo DB using the database schema"
+
+  def generate_mongo_dsl
+
+    configuration_hash = {:host => options["host"],
+                          :port => options["port"],
+                          :database => options["database"],
+                          :username => options["username"],
+                          :password => options["password"]
+    }
+
+    create_file "mongodb_whitelist_generated.rb"
+    DataAnon::ThorHelpers::MongoDBDSLGenerator.new(configuration_hash, options["whitelist_patterns"]).generate
+  end
+
+  method_option :host, :required => true, :aliases => "-h", :desc => "Source Database host [required]", :for => :generate_mongo_dsl
+  method_option :database, :required => true, :aliases => "-d", :desc => "Database name [required]", :for => :generate_mongo_dsl
+  method_option :port, :aliases => "-p", :desc => "Port to connect to. If not provided default port will be used", :for => :generate_mongo_dsl
+  method_option :username, :aliases => "-u", :desc => "Username", :for => :generate_mongo_dsl
+  method_option :password, :aliases => "-w", :desc => "Password", :for => :generate_mongo_dsl
+  method_option :whitelist_patterns, :aliases => "-r", :desc => "Whitelist Patterns", :for => :generate_mongo_dsl
+
+end
+
+AnonymizationCLI.start

data/data-anonymization.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.homepage      = "http://sunitparekh.github.com/data-anonymization"
 
   gem.files         = `git ls-files`.split($/).select { |f| !f.match(/^sample-data/) }
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.executables   = "datanon"
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
@@ -24,4 +24,5 @@ Gem::Specification.new do |gem|
   gem.add_dependency('rgeo-geojson', '~> 0.2.3')
   gem.add_dependency('powerbar', '~> 1.0.8')
   gem.add_dependency('parallel', '~> 0.5.18')
+  gem.add_dependency('thor', '~> 0.16.0')
 end

data/examples/blacklist_dsl.rb

@@ -0,0 +1,42 @@
+system "bundle exec ruby whitelist_dsl.rb"
+
+require 'data-anonymization'
+
+DataAnon::Utils::Logging.logger.level = Logger::INFO
+
+database 'Chinook' do
+  strategy DataAnon::Strategy::Blacklist
+  source_db :adapter => 'sqlite3', :database => 'sample-data/chinook-empty.sqlite'
+
+  table 'Employee' do
+    primary_key 'EmployeeId'
+    anonymize('BirthDate').using FieldStrategy::DateTimeDelta.new(1, 1)
+    anonymize('FirstName').using FieldStrategy::RandomFirstName.new
+    anonymize('LastName').using FieldStrategy::RandomLastName.new
+    anonymize('HireDate').using FieldStrategy::DateTimeDelta.new(2, 0)
+    anonymize('Address').using FieldStrategy::RandomAddress.region_US
+    anonymize('City').using FieldStrategy::RandomCity.region_US
+    anonymize('State').using FieldStrategy::RandomProvince.region_US
+    anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
+    anonymize('Country') {|field| "USA" }
+    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+  end
+
+  table 'Customer' do
+    primary_key 'CustomerId'
+    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('FirstName').using FieldStrategy::RandomFirstName.new
+    anonymize('LastName').using FieldStrategy::RandomLastName.new
+    anonymize('Address').using FieldStrategy::RandomAddress.region_US
+    anonymize('City').using FieldStrategy::RandomCity.region_US
+    anonymize('State').using FieldStrategy::RandomProvince.region_US
+    anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
+    anonymize('Country') {|field| "USA" }
+    anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+  end
+
+end
+

data/examples/mongodb_blacklist_dsl.rb

@@ -0,0 +1,38 @@
+require 'data-anonymization'
+
+require 'mongo'
+Mongo::Connection.from_uri("mongodb://localhost/test").drop_database('test')
+system "mongoimport -d test --drop -c users --jsonArray ./sample-data/mongo/users.json"
+system "mongoimport -d test --drop -c plans --jsonArray ./sample-data/mongo/plans.json"
+
+DataAnon::Utils::Logging.logger.level = Logger::INFO
+
+database 'test' do
+  strategy DataAnon::Strategy::MongoDB::Blacklist
+  source_db :mongodb_uri => "mongodb://localhost/test", :database => 'test'
+
+  collection 'users' do
+    anonymize('date_of_birth').using FieldStrategy::TimeDelta.new(5,30)
+    anonymize('user_id').using FieldStrategy::StringTemplate.new('user-#{row_number}')
+    anonymize('email').using FieldStrategy::RandomMailinatorEmail.new
+    anonymize('password') { |field| "password" }
+    anonymize('first_name').using FieldStrategy::RandomFirstName.new
+    anonymize('last_name').using FieldStrategy::RandomLastName.new
+  end
+
+  collection 'plans' do
+    anonymize('plan_aliases').using FieldStrategy::SelectFromList.new(["Free","Team","Business","Paid"])
+    anonymize 'public_sharing','photo_sharing'
+
+    collection 'features' do
+      anonymize('max_storage').using FieldStrategy::SelectFromList.new([10737418240,21474836480,53687091200])
+
+      document 'users' do
+        anonymize 'max', 'additional'
+      end
+    end
+
+  end
+
+end
+

data/examples/mongodb_whitelist_dsl.rb

@@ -0,0 +1,44 @@
+require 'data-anonymization'
+
+require 'mongo'
+Mongo::Connection.from_uri("mongodb://localhost/test").drop_database('test')
+Mongo::Connection.from_uri("mongodb://localhost/dest").drop_database('dest')
+system "mongoimport -d test --drop -c users --jsonArray ./sample-data/mongo/users.json"
+system "mongoimport -d test --drop -c plans --jsonArray ./sample-data/mongo/plans.json"
+
+DataAnon::Utils::Logging.logger.level = Logger::INFO
+
+database 'test' do
+  strategy DataAnon::Strategy::MongoDB::Whitelist
+  source_db :mongodb_uri => "mongodb://localhost/test", :database => 'test'
+  destination_db :mongodb_uri => "mongodb://localhost/dest", :database => 'dest'
+
+  collection 'users' do
+    whitelist '_id','failed_attempts','updated_at'
+    anonymize('date_of_birth').using FieldStrategy::TimeDelta.new(5,30)
+    anonymize('user_id').using FieldStrategy::StringTemplate.new('user-#{row_number}')
+    anonymize('email').using FieldStrategy::RandomMailinatorEmail.new
+    anonymize('password') { |field| "password" }
+    anonymize('first_name').using FieldStrategy::RandomFirstName.new
+    anonymize('last_name').using FieldStrategy::RandomLastName.new
+    anonymize 'password_reset_answer','password_reset_question'
+  end
+
+  collection 'plans' do
+    whitelist '_id', 'name','term', 'created_at'
+    anonymize('plan_aliases').using FieldStrategy::SelectFromList.new(["Free","Team","Business","Paid"])
+    anonymize 'public_sharing','photo_sharing'
+
+    collection 'features' do
+      anonymize('max_storage').using FieldStrategy::SelectFromList.new([10737418240,21474836480,53687091200])
+      whitelist 'type'
+
+      document 'users' do
+        anonymize 'max', 'additional'
+      end
+    end
+
+  end
+
+end
+

data/examples/whitelist_dsl.rb

@@ -0,0 +1,63 @@
+system "rake empty_dest" # clean destination database on every call
+
+require 'data-anonymization'
+
+DataAnon::Utils::Logging.logger.level = Logger::INFO
+
+database 'Chinook' do
+  strategy DataAnon::Strategy::Whitelist
+  source_db :adapter => 'sqlite3', :database => 'sample-data/chinook.sqlite'
+  destination_db :adapter => 'sqlite3', :database => 'sample-data/chinook-empty.sqlite'
+
+  default_field_strategies :string => FieldStrategy::StringTemplate.new('Sunit #{row_number} Parekh')
+
+  table 'Genre' do
+    primary_key 'GenreId'
+    whitelist 'GenreId'
+    anonymize 'Name' do |field|
+      field.value + " test"
+    end
+  end
+
+  table 'MediaType' do
+    primary_key 'MediaTypeId'
+    anonymize('MediaTypeId') { |field| field.value } # same as whitelist
+    anonymize('Name').using FieldStrategy::StringTemplate.new('Media Type #{row_number}')
+
+  end
+
+  table 'Employee' do
+    primary_key 'EmployeeId'
+    whitelist 'EmployeeId', 'ReportsTo', 'Title'
+    anonymize('BirthDate').using FieldStrategy::DateTimeDelta.new(1, 1)
+    anonymize('FirstName').using FieldStrategy::RandomFirstName.new
+    anonymize('LastName').using FieldStrategy::RandomLastName.new
+    anonymize('HireDate').using FieldStrategy::DateTimeDelta.new(2, 0)
+    anonymize('Address').using FieldStrategy::RandomAddress.region_US
+    anonymize('City').using FieldStrategy::RandomCity.region_US
+    anonymize('State').using FieldStrategy::RandomProvince.region_US
+    anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
+    anonymize('Country') {|field| "USA" }
+    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+  end
+
+  table 'Customer' do
+    primary_key 'CustomerId'
+    whitelist 'SupportRepId', 'Company'
+    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('FirstName').using FieldStrategy::RandomFirstName.new
+    anonymize('LastName').using FieldStrategy::RandomLastName.new
+    anonymize('Address').using FieldStrategy::RandomAddress.region_US
+    anonymize('City').using FieldStrategy::RandomCity.region_US
+    anonymize('State').using FieldStrategy::RandomProvince.region_US
+    anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
+    anonymize('Country') {|field| "USA" }
+    anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+  end
+
+
+end
+

data/lib/core/database.rb

@@ -2,6 +2,7 @@ module DataAnon
   module Core
 
     class Database
+      include Utils::Logging
 
       def initialize name
         @name = name
@@ -35,18 +36,35 @@ module DataAnon
 
       def table (name, &block)
         table = @strategy.new(@source_database, @destination_database, name, @user_defaults).process_fields(&block)
-        @tables<< table
+        @tables << table
       end
+      alias :collection :table
 
       def anonymize
-        @execution_strategy.new.anonymize @tables
+        begin
+          @execution_strategy.new.anonymize @tables
+        rescue => e
+          logger.error "\n#{e.message} \n #{e.backtrace}"
+        end
+        if @strategy.whitelist?
+          logger.info("Fields missing the anonymization strategy")
+          @tables.each { |table| table.fields_missing_strategy.print }
+        end
+
+        @tables.each { |table| table.errors.print }
       end
 
     end
 
     class Sequential
       def anonymize tables
-        tables.each { |table| table.process }
+        tables.each do |table|
+          begin
+            table.process
+          rescue => e
+            logger.error "\n#{e.message} \n #{e.backtrace}"
+          end
+        end
       end
     end
 

data/lib/core/field.rb

@@ -3,14 +3,17 @@ module DataAnon
 
     class Field
 
-      def initialize name, value, row_number, ar_record
+      def initialize name, value, row_number, ar_record, table_name = "unknown"
         @name = name
         @value = value
         @row_number = row_number
         @ar_record = ar_record
+        @table_name = table_name
       end
 
-      attr_accessor :name, :value, :row_number, :ar_record
+      attr_accessor :name, :value, :row_number, :ar_record, :table_name
+
+      alias :collection_name :table_name
 
     end
 

data/lib/core/fields_missing_strategy.rb

@@ -0,0 +1,30 @@
+module DataAnon
+  module Core
+
+    class FieldsMissingStrategy
+      include Utils::Logging
+
+      def initialize table_name
+        @table_name = table_name
+        @fields_missing_strategy = []
+      end
+
+      def missing field_name
+        return if @fields_missing_strategy.include? field_name
+        @fields_missing_strategy << field_name
+      end
+
+      def fields_missing_strategy
+        @fields_missing_strategy
+      end
+
+      def print
+        @fields_missing_strategy.each do |field_name|
+          logger.info("#{@table_name}.#{field_name}")
+        end
+      end
+
+    end
+
+  end
+end

data/lib/core/table_errors.rb

@@ -0,0 +1,32 @@
+module DataAnon
+  module Core
+
+    class TableErrors
+      include Utils::Logging
+
+      def initialize table_name
+        @table_name = table_name
+        @errors = []
+      end
+
+      def log_error record, exception
+        @errors << { :record => record, :exception => exception}
+        raise "Reached limit of error for a table" if @errors.length > 100
+      end
+
+      def errors
+        @errors
+      end
+
+      def print
+        return if @errors.length == 0
+        logger.error("Errors while processing table #{@table_name}:")
+        @errors.each do |error|
+          logger.error(error[:exception])
+        end
+      end
+
+    end
+
+  end
+end

data/lib/data-anonymization.rb

@@ -7,11 +7,22 @@ require "utils/random_string"
 require "utils/random_string_chars_only"
 require "utils/geojson_parser"
 require "utils/progress_bar"
+require "utils/parallel_progress_bar"
 require "utils/resource"
+require "utils/template_helper"
 require "parallel/table"
 require "core/database"
+require "core/fields_missing_strategy"
+require "thor/helpers/rdbms_dsl_generator"
 require "core/field"
+require "core/table_errors"
 require "strategy/strategies"
 require "utils/database"
 require "core/dsl"
 
+begin
+  require 'mongo'
+  require "thor/helpers/mongodb_dsl_generator"
+rescue LoadError
+  "Ignoring the mongodb specific libraries if monog driver is not specified in gem"
+end

data/lib/parallel/table.rb

@@ -5,7 +5,14 @@ module DataAnon
     class Table
 
       def anonymize tables
-        ::Parallel.each(tables) { |table| table.process }
+        ::Parallel.each(tables) do |table|
+          begin
+            table.progress_bar_class DataAnon::Utils::ParallelProgressBar
+            table.process
+          rescue => e
+            logger.error "\n#{e.message} \n #{e.backtrace}"
+          end
+        end
       end
 
     end

data/lib/strategy/base.rb

@@ -3,12 +3,20 @@ module DataAnon
     class Base
       include Utils::Logging
 
+      attr_accessor :fields, :user_strategies, :fields_missing_strategy, :errors
+
       def initialize source_database, destination_database, name, user_strategies
         @name = name
         @user_strategies = user_strategies
         @fields = {}
         @source_database = source_database
         @destination_database = destination_database
+        @fields_missing_strategy = DataAnon::Core::FieldsMissingStrategy.new name
+        @errors = DataAnon::Core::TableErrors.new(@name)
+      end
+
+      def self.whitelist?
+        false
       end
 
       def process_fields &block
@@ -20,19 +28,10 @@ module DataAnon
         @primary_keys = fields
       end
 
-      def is_primary_key? field
-        @primary_keys.select { |key| field.downcase == key.downcase }.length > 0
-      end
-
-
       def whitelist *fields
         fields.each { |f| @fields[f.downcase] = DataAnon::Strategy::Field::Whitelist.new }
       end
 
-      def fields
-        @fields
-      end
-
       def anonymize *fields, &block
         if block.nil?
           fields.each { |f| @fields[f.downcase] = DataAnon::Strategy::Field::DefaultAnon.new(@user_strategies) }
@@ -49,6 +48,15 @@ module DataAnon
         end
       end
 
+      def is_primary_key? field
+        @primary_keys.select { |key| field.downcase == key.downcase }.length > 0
+      end
+
+      def default_strategy field_name
+        @fields_missing_strategy.missing field_name
+        DataAnon::Strategy::Field::DefaultAnon.new(@user_strategies)
+      end
+
       def dest_table
         return @dest_table unless @dest_table.nil?
         DataAnon::Utils::DestinationDatabase.establish_connection @destination_database if @destination_database
@@ -65,17 +73,30 @@ module DataAnon
         logger.debug "Processing table #{@name} with fields strategies #{@fields}"
         total = source_table.count
         if total > 0
-          index = 1
-          progress_bar = DataAnon::Utils::ProgressBar.new @name, total
+          index = 0
+          progress = progress_bar.new(@name, total)
           source_table.all.each do |record|
-            process_record index, record
             index += 1
-            progress_bar.show(index)
+            begin
+              process_record index, record
+            rescue => exception
+              @errors.log_error record, exception
+            end
+            progress.show index
           end
-          progress_bar.close
+          progress.close
         end
       end
 
+      def progress_bar
+        @progress_bar || DataAnon::Utils::ProgressBar
+      end
+
+      def progress_bar_class progress_bar
+        @progress_bar = progress_bar
+      end
+
+
     end
   end
 end