data-anonymization 0.5.0 → 0.5.1.rc1

data/README.md

@@ -68,14 +68,14 @@ Postgresql database having **composite primary key**
 
 ## Changelog
 
-#### 0.5.0 (rc1 released. install gem using --pre option)
+#### 0.5.0  (Sep 28, 2012)
 
 Major changes:
 
 1. MongoDB support
 2. Command line utility to generate whitelist DSL for RDBMS & MongoDB (reduces pain for writing whitelist dsl)
 3. Added support for reporting fields missing mapping in case of whitelist
-4. Errors reported at the end of process. Job doesn't fail for a single error.
+4. Errors reported at the end of process. Job doesn't fail for a single error, it fails it more than 100 records failed during anonymization.
 
 
 Please see the [Github 0.5.0 milestone page](https://github.com/sunitparekh/data-anonymization/issues?milestone=2&state=open) for more details on changes/fixes in release 0.5.0
@@ -115,6 +115,8 @@ For almost all projects there is a need for production data dump in order to run
 However, getting production data and using it is not feasible due to multiple reasons, primary being privacy concerns for user data. And thus the need for data anonymization.
 This tool helps you to get anonymized production data dump using either Blacklist or Whitelist strategies.
 
+Read more about [data anonymization here](http://sunitspace.blogspot.in/2012/09/data-anonymization.html)
+
 ## Anonymization Strategies
 
 ### Blacklist
@@ -147,6 +149,9 @@ database 'DatabaseName' do
 end
 ```
 
+Read more about [blacklist and whitelist here](http://sunitspace.blogspot.in/2012/09/data-anonymization-blacklist-whitelist.html)
+
+
 ## Tips
 
 1. In Whitelist approach make source database connection READONLY.

data/examples/whitelist_dsl.rb

@@ -26,38 +26,42 @@ database 'Chinook' do
 
   end
 
-  table 'Employee' do
-    primary_key 'EmployeeId'
-    whitelist 'EmployeeId', 'ReportsTo', 'Title'
-    anonymize('BirthDate').using FieldStrategy::DateTimeDelta.new(1, 1)
+  table 'Customer' do
+    primary_key 'CustomerId'
+    batch_size 5  # batch_size works only if the primary_key is defined for the table
+
+    whitelist 'CustomerId', 'SupportRepId', 'Company'
+    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
     anonymize('FirstName').using FieldStrategy::RandomFirstName.new
     anonymize('LastName').using FieldStrategy::RandomLastName.new
-    anonymize('HireDate').using FieldStrategy::DateTimeDelta.new(2, 0)
     anonymize('Address').using FieldStrategy::RandomAddress.region_US
     anonymize('City').using FieldStrategy::RandomCity.region_US
     anonymize('State').using FieldStrategy::RandomProvince.region_US
     anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
     anonymize('Country') {|field| "USA" }
-    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
   end
 
-  table 'Customer' do
-    primary_key 'CustomerId'
-    whitelist 'SupportRepId', 'Company'
-    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
+  table 'Employee' do
+    batch_size 5  # this won't work since there is no 'primary_key' defined
+
+    whitelist 'EmployeeId', 'ReportsTo', 'Title'
+    anonymize('BirthDate').using FieldStrategy::DateTimeDelta.new(1, 1)
     anonymize('FirstName').using FieldStrategy::RandomFirstName.new
     anonymize('LastName').using FieldStrategy::RandomLastName.new
+    anonymize('HireDate').using FieldStrategy::DateTimeDelta.new(2, 0)
     anonymize('Address').using FieldStrategy::RandomAddress.region_US
     anonymize('City').using FieldStrategy::RandomCity.region_US
     anonymize('State').using FieldStrategy::RandomProvince.region_US
     anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
     anonymize('Country') {|field| "USA" }
+    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
   end
 
 
+
 end
 

data/lib/core/table_errors.rb

@@ -20,9 +20,10 @@ module DataAnon
 
       def print
         return if @errors.length == 0
-        logger.error("Errors while processing table #{@table_name}:")
+        logger.error("Errors while processing table '#{@table_name}':")
         @errors.each do |error|
           logger.error(error[:exception])
+          logger.error(error[:exception].backtrace.join("\n\t"))
         end
       end
 

data/lib/strategy/base.rb

@@ -13,6 +13,7 @@ module DataAnon
         @destination_database = destination_database
         @fields_missing_strategy = DataAnon::Core::FieldsMissingStrategy.new name
         @errors = DataAnon::Core::TableErrors.new(@name)
+        @primary_keys = []
       end
 
       def self.whitelist?
@@ -28,6 +29,10 @@ module DataAnon
         @primary_keys = fields
       end
 
+      def batch_size size
+        @batch_size = size
+      end
+
       def whitelist *fields
         fields.each { |f| @fields[f.downcase] = DataAnon::Strategy::Field::Whitelist.new }
       end
@@ -73,21 +78,43 @@ module DataAnon
         logger.debug "Processing table #{@name} with fields strategies #{@fields}"
         total = source_table.count
         if total > 0
-          index = 0
           progress = progress_bar.new(@name, total)
-          source_table.all.each do |record|
-            index += 1
-            begin
-              process_record index, record
-            rescue => exception
-              @errors.log_error record, exception
-            end
-            progress.show index
+          if @primary_keys.empty? || !@batch_size.present?
+            process_table progress
+          else
+            process_table_in_batches progress
           end
           progress.close
         end
       end
 
+      def process_table progress
+        index = 0
+        source_table.all.each do |record|
+          index += 1
+          begin
+            process_record index, record
+          rescue => exception
+            @errors.log_error record, exception
+          end
+          progress.show index
+        end
+      end
+
+      def process_table_in_batches progress
+        logger.info "Processing table #{@name} records in batch size of #{@batch_size}"
+        index = 0
+        source_table.find_each(:batch_size => @batch_size) do |record|
+          index += 1
+          begin
+            process_record index, record
+          rescue => exception
+            @errors.log_error record, exception
+          end
+          progress.show index
+        end
+      end
+
       def progress_bar
         @progress_bar || DataAnon::Utils::ProgressBar
       end

data/lib/utils/database.rb

@@ -28,11 +28,12 @@ module DataAnon
 
     class BaseTable
 
-      def self.create_table  database, table_name, primary_keys
+      def self.create_table  database, table_name, primary_keys = []
         Class.new(database) do
           self.table_name = table_name
           self.primary_keys = primary_keys if primary_keys.length > 1
           self.primary_key = primary_keys[0] if primary_keys.length == 1
+          self.primary_key = nil if primary_keys.length == 0
           self.inheritance_column = :_type_disabled
           self.mass_assignment_sanitizer = MassAssignmentIgnoreSanitizer.new(self)
         end
@@ -42,7 +43,7 @@ module DataAnon
 
     class SourceTable < BaseTable
 
-      def self.create table_name, primary_key
+      def self.create table_name, primary_key = []
         create_table  SourceDatabase, table_name, primary_key
       end
 
@@ -50,7 +51,7 @@ module DataAnon
 
     class DestinationTable < BaseTable
 
-      def self.create table_name, primary_key
+      def self.create table_name, primary_key = []
         create_table DestinationDatabase, table_name, primary_key
       end
 

data/lib/version.rb

@@ -1,3 +1,3 @@
 module DataAnonymization
-  VERSION = "0.5.0"
+  VERSION = "0.5.1.rc1"
 end

data/spec/acceptance/rdbms_whitelist_spec.rb

@@ -21,7 +21,6 @@ describe "End 2 End RDBMS Whitelist Acceptance Test using SQLite database" do
       destination_db dest_connection_spec
 
       table 'customers' do
-        primary_key 'cust_id'
         whitelist 'cust_id', 'address', 'zipcode', 'blog_url'
         anonymize('first_name').using FieldStrategy::RandomFirstName.new
         anonymize('last_name').using FieldStrategy::RandomLastName.new
@@ -34,8 +33,8 @@ describe "End 2 End RDBMS Whitelist Acceptance Test using SQLite database" do
     end
 
     DataAnon::Utils::DestinationDatabase.establish_connection dest_connection_spec
-    dest_table = DataAnon::Utils::DestinationTable.create 'customers', ['cust_id']
-    new_rec = dest_table.find(CustomerSample::SAMPLE_DATA[:cust_id])
+    dest_table = DataAnon::Utils::DestinationTable.create 'customers'
+    new_rec = dest_table.where("cust_id" => CustomerSample::SAMPLE_DATA[:cust_id]).first
     new_rec.first_name.should_not be("Sunit")
     new_rec.last_name.should_not be("Parekh")
     new_rec.birth_date.should_not be(Date.new(1977,7,8))

data/spec/acceptance/rdbms_whitelist_with_primary_key_spec.rb

@@ -0,0 +1,55 @@
+require "spec_helper"
+
+describe "End 2 End RDBMS Whitelist Acceptance Test using SQLite database" do
+
+  source_connection_spec = {:adapter => 'sqlite3', :database => 'tmp/customer.sqlite'}
+  dest_connection_spec = {:adapter => 'sqlite3', :database => 'tmp/customer-dest.sqlite'}
+
+  before(:each) do
+    CustomerSample.clean
+    CustomerSample.create_schema source_connection_spec
+    CustomerSample.insert_record source_connection_spec, CustomerSample::SAMPLE_DATA
+
+    CustomerSample.create_schema dest_connection_spec
+  end
+
+  it "should anonymize customer table record " do
+
+    database "Customer" do
+      strategy DataAnon::Strategy::Whitelist
+      source_db source_connection_spec
+      destination_db dest_connection_spec
+
+      table 'customers' do
+        primary_key 'cust_id'
+        batch_size 1
+
+        whitelist 'cust_id', 'address', 'zipcode', 'blog_url'
+        anonymize('first_name').using FieldStrategy::RandomFirstName.new
+        anonymize('last_name').using FieldStrategy::RandomLastName.new
+        anonymize('state').using FieldStrategy::SelectFromList.new(['Gujrat','Karnataka'])
+        anonymize('phone').using FieldStrategy::RandomPhoneNumber.new
+        anonymize('email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+        anonymize 'terms_n_condition', 'age', 'longitude'
+        anonymize('latitude').using FieldStrategy::RandomFloatDelta.new(2.0)
+      end
+    end
+
+    DataAnon::Utils::DestinationDatabase.establish_connection dest_connection_spec
+    dest_table = DataAnon::Utils::DestinationTable.create 'customers'
+    new_rec = dest_table.where("cust_id" => CustomerSample::SAMPLE_DATA[:cust_id]).first
+    new_rec.first_name.should_not be("Sunit")
+    new_rec.last_name.should_not be("Parekh")
+    new_rec.birth_date.should_not be(Date.new(1977,7,8))
+    new_rec.address.should == 'F 501 Shanti Nagar'
+    ['Gujrat','Karnataka'].should include(new_rec.state)
+    new_rec.zipcode.should == '411048'
+    new_rec.phone.should_not be "9923700662"
+    new_rec.email.should == 'test+1@gmail.com'
+    [true,false].should include(new_rec.terms_n_condition)
+    new_rec.age.should be_between(0,100)
+    new_rec.latitude.should be_between( 38.689060, 42.689060)
+    new_rec.longitude.should be_between( -84.044636, -64.044636)
+
+  end
+end

data/spec/support/customer_sample.rb

@@ -2,7 +2,7 @@ class CustomerSample
 
   class CreateCustomer < ActiveRecord::Migration
     def up
-      create_table :customers, { :id => false } do |t|
+      create_table :customers, { :id => false, :force => true } do |t|
         t.integer :cust_id, :primary => true
         t.string :first_name
         t.string :last_name

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: data-anonymization
 version: !ruby/object:Gem::Version
-  version: 0.5.0
-  prerelease: 
+  version: 0.5.1.rc1
+  prerelease: 6
 platform: ruby
 authors:
 - Sunit Parekh
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-28 00:00:00.000000000 Z
+date: 2012-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -244,6 +244,7 @@ files:
 - spec/acceptance/mongodb_whitelist_spec.rb
 - spec/acceptance/rdbms_blacklist_spec.rb
 - spec/acceptance/rdbms_whitelist_spec.rb
+- spec/acceptance/rdbms_whitelist_with_primary_key_spec.rb
 - spec/core/fields_missing_strategy_spec.rb
 - spec/resource/sample.geojson
 - spec/spec_helper.rb
@@ -306,9 +307,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -321,6 +322,7 @@ test_files:
 - spec/acceptance/mongodb_whitelist_spec.rb
 - spec/acceptance/rdbms_blacklist_spec.rb
 - spec/acceptance/rdbms_whitelist_spec.rb
+- spec/acceptance/rdbms_whitelist_with_primary_key_spec.rb
 - spec/core/fields_missing_strategy_spec.rb
 - spec/resource/sample.geojson
 - spec/spec_helper.rb