data-anonymization 0.7.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 05181aca5f0046f53ca999bbe73aea0360f013b6
-  data.tar.gz: d538308e3ccb427af3860b657362f3de2e6aff5b
+  metadata.gz: afcaf064c0b8135149e5fb8e559ab95df8088516
+  data.tar.gz: 795dbba88fead534acdcb87b4cf18fd4bdf12ad8
 SHA512:
-  metadata.gz: 83a63960a9dcd64cd74ba4aa01117f94f813a118af79b5e9ca57eb333fbc143699b07ba801909ff22f03e276ff268b84b6b2c70030ce3eaa06526eb3abacb91e
-  data.tar.gz: 25fc3f8467d4ce3dc6c0ea455a9fa1660113f0a94e2a39edf48e953d87af3519cf03c0a253c8e7d23b6beb4e195f033f3664d0a70bf83b2d1fa89e86f3bb30e4
+  metadata.gz: 85daa8177667c232eae29f3ad9a9f021a5d679898679967a5d435d4b3e3b7eb9bf0a6e84cd5441b43d823005a9dc0c295adfa3b9ca9387e1593ff01f4dc950d2
+  data.tar.gz: 2ce7411e57e5427e9fbcd211b37a60acc3669d38a2405364e51aeb27f5ff2d43c027df5e55291e8be39f81ad4543045c318a5d20422c0444780d912da9595c1d

data/README.md

@@ -70,6 +70,11 @@ Postgresql database having **composite primary key**
 
 ## Changelog
 
+#### 0.7.1 (Jun 13, 2015)
+1. Fixed issues with empty array data for MongoDB
+2. Added feature to skip and continue records during anaonymisation, this is useful to apply different strategies for different types of records.
+
+
 #### 0.7.0 (Mar 9, 2015)
 1. Removed downcase from field name since it was causing issues with upper case field names. So now for databsae where case matters field name case should be maintained. 
 2. Upgraded gems to latest version
@@ -200,6 +205,7 @@ Read more about [blacklist and whitelist here](http://sunitspace.blogspot.in/201
 3. To run anonymization in parallel at Table level, provided no FK constraint on tables use DataAnon::Parallel::Table strategy
 4. For large table to load them in batches from table set 'batch_size' and it will use RoR's batch mode processing. Checkout [example](https://github.com/sunitparekh/data-anonymization/blob/master/examples/whitelist_dsl.rb) on how to use batch processing.
 5. Make sure to give proper case for fields and table names. 
+6. Use skip and continue to apply different strategies for records. 
 
 ## DSL Generation
 
@@ -567,6 +573,37 @@ ENV['show_progress'] = 'false'
 DataAnon::Utils::Logging.logger.level = Logger::INFO
 ```
 
+## Skip and Continue records
+
+*Skip* is used to skip records during anonymization when condition returns true. This records are ignored, 
+in blacklist it remains as it is in database and in case of whitelist this records will not be copied to destination database.
+
+```ruby
+table 'customers' do
+  skip { |index, record| record['age'] < 18 }
+
+  primary_key 'cust_id'
+  anonymize('email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+  anonymize 'terms_n_condition', 'age'
+end
+```
+  
+  
+*Continue* is exactly opposite of Skip and it continue with anonymization only if given condition returns true. 
+In case of blacklist records are anonymized for matching conditions and for whitelist records are anonymized and copied
+to new database for matching conditions. 
+
+```ruby
+table 'customers' do
+  continue { |index, record| record['age'] > 18 }
+
+  primary_key 'cust_id'
+  anonymize('email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+  anonymize 'terms_n_condition', 'age'
+end
+```
+
+
 ## Want to contribute?
 
 1. Fork it

data/examples/mongodb_blacklist_dsl.rb

@@ -21,10 +21,10 @@ database 'test' do
   end
 
   collection 'plans' do
-    anonymize('plan_aliases').using FieldStrategy::SelectFromList.new(["Free","Team","Business","Paid"])
+    anonymize('plan_aliases').using FieldStrategy::AnonymizeArray.new(FieldStrategy::SelectFromList.new(["Free","Team","Business","Paid"]))
     anonymize 'public_sharing','photo_sharing'
 
-    collection 'features' do
+    document 'features' do
       anonymize('max_storage').using FieldStrategy::SelectFromList.new([10737418240,21474836480,53687091200])
 
       document 'users' do

data/examples/mongodb_whitelist_dsl.rb

@@ -26,7 +26,7 @@ database 'test' do
 
   collection 'plans' do
     whitelist '_id', 'name','term', 'created_at'
-    anonymize('plan_aliases').using FieldStrategy::SelectFromList.new(["Free","Team","Business","Paid"])
+    anonymize('plan_aliases').using FieldStrategy::AnonymizeArray.new(FieldStrategy::SelectFromList.new(["Free","Team","Business","Paid"]))
     anonymize 'public_sharing','photo_sharing'
 
     collection 'features' do

data/lib/strategy/base.rb

@@ -37,6 +37,14 @@ module DataAnon
         fields.each { |f| @fields[f] = DataAnon::Strategy::Field::Whitelist.new }
       end
 
+      def skip &block
+        @skip_block = block
+      end
+
+      def continue &block
+        @continue_block = block
+      end
+
       def anonymize *fields, &block
         if block.nil?
           fields.each { |f| @fields[f] = DataAnon::Strategy::Field::DefaultAnon.new(@user_strategies) }
@@ -93,7 +101,7 @@ module DataAnon
         source_table.all.each do |record|
           index += 1
           begin
-            process_record index, record
+            process_record_if index, record
           rescue => exception
             @errors.log_error record, exception
           end
@@ -107,7 +115,7 @@ module DataAnon
         source_table.find_each(:batch_size => @batch_size) do |record|
           index += 1
           begin
-            process_record index, record
+            process_record_if index, record
           rescue => exception
             @errors.log_error record, exception
           end
@@ -115,6 +123,13 @@ module DataAnon
         end
       end
 
+      def process_record_if index, record
+        return if @skip_block && @skip_block.call(index, record)
+        return if @continue_block && !@continue_block.call(index, record)
+
+        process_record index, record
+      end
+
       def progress_bar
         @progress_bar || DataAnon::Utils::ProgressBar
       end

data/lib/strategy/mongodb/anonymize_field.rb

@@ -30,11 +30,11 @@ module DataAnon
         end
 
         def sub_documents?
-          @field.value.kind_of?(Array) && @field.value[0].kind_of?(Hash)
+          @field.value.kind_of?(Array) && (@field_strategy.kind_of?(Hash) || @field.value[0].kind_of?(Hash))
         end
 
         def sub_document?
-          @field.value.kind_of? Hash
+          @field.value.kind_of?(Hash)
         end
 
 

data/lib/version.rb

@@ -1,3 +1,3 @@
 module DataAnonymization
-  VERSION = '0.7.0'
+  VERSION = '0.7.1'
 end

data/spec/acceptance/mongodb_blacklist_spec.rb

@@ -10,27 +10,30 @@ describe "End 2 End MongoDB Blacklist Acceptance Test" do
             "_id" => 1,
             "USER_ID" => "sunitparekh",
             "date_of_birth" => Time.new(2012, 7, 14, 13, 1, 0),
-            "email" => "parekh.sunit@gmail.com",
+            "email" => "parekh-sunit@mailinator.com",
             "password" => "TfqIK8Pd8GlbMDFZCX4l/5EtnOkfLCeynOL85tJQuxum&382knaflk@@",
             "failed_attempts" => 0,
             "first_name" => "Sunit",
             "last_name" => "Parekh",
             "password_reset_answer" => "manza",
             "password_reset_question" => "My new car modal?",
-            "updated_at" => Time.new(2012, 8, 15, 13, 1, 0)
+            "updated_at" => Time.new(2012, 8, 15, 13, 1, 0),
+            "alternate_emails" => ["abc@test.com","abc2@test.com"]
+
         },
         {
             "_id" => 2,
-            "user_id" => "anandagrawal",
+            "USER_ID" => "anandagrawal",
             "date_of_birth" => Time.new(2011, 8, 11, 13, 1, 0),
-            "email" => "anandagrawal84@gmail.com",
+            "email" => "anand-agrawal@mailinator.com",
             "password" => "Tz548O0RWusldVAWkwqfzO3jK/X4l/5EtnOkfLCeynOL85tJQuxum",
             "failed_attempts" => 0,
             "first_name" => "Anand",
             "last_name" => "Agrawal",
             "password_reset_answer" => "android",
             "password_reset_question" => "My phone?",
-            "updated_at" => Time.new(2012, 2, 11, 13, 1, 0)
+            "updated_at" => Time.new(2012, 2, 11, 13, 1, 0),
+            "alternate_emails" => ["abc@test.com","abc2@test.com"]
         }
     ]
     users_coll = Mongo::Connection.from_uri("mongodb://localhost/dest")['test']['users']
@@ -50,6 +53,7 @@ describe "End 2 End MongoDB Blacklist Acceptance Test" do
         anonymize('password') { |field| "password" }
         anonymize('first_name').using FieldStrategy::RandomFirstName.new
         anonymize('last_name').using FieldStrategy::RandomLastName.new
+        anonymize('alternate_emails').using FieldStrategy::AnonymizeArray.new(FieldStrategy::RandomMailinatorEmail.new)
       end
 
     end
@@ -61,7 +65,7 @@ describe "End 2 End MongoDB Blacklist Acceptance Test" do
     user['_id'].should == 1
     user['USER_ID'].should == "user-1"
     user['date_of_birth'].to_i.should_not == Time.new(2012, 7, 14, 13, 1, 0).to_i
-    user['email'].should_not == "parekh.sunit@gmail.com"
+    user['email'].should_not == "parekh-sunit@mailinator.com"
     user['password'].should == "password"
     user['failed_attempts'].should == 0
     user['first_name'].should_not be "Sunit"
@@ -69,6 +73,9 @@ describe "End 2 End MongoDB Blacklist Acceptance Test" do
     user['password_reset_answer'].should == "manza"
     user['password_reset_question'].should == "My new car modal?"
     user['updated_at'].to_i.should == Time.new(2012, 8, 15, 13, 1, 0).to_i
+    user['alternate_emails'].length.should == 2
+    user['alternate_emails'][0].should_not == 'abc@test.com'
+    user['alternate_emails'][1].should_not == 'abc2@test.com'
 
 
   end

data/spec/acceptance/rdbms_blacklist_spec.rb

@@ -1,6 +1,6 @@
-require "spec_helper"
+require 'spec_helper'
 
-describe "End 2 End RDBMS Blacklist Acceptance Test using SQLite database" do
+describe 'End 2 End RDBMS Blacklist Acceptance Test using SQLite database' do
   connection_spec = {:adapter => 'sqlite3', :database => 'tmp/customer.sqlite'}
 
   before(:each) do
@@ -9,9 +9,9 @@ describe "End 2 End RDBMS Blacklist Acceptance Test using SQLite database" do
     CustomerSample.insert_record connection_spec, CustomerSample::SAMPLE_DATA
   end
 
-  it "should anonymize customer table record " do
+  it 'should anonymize customer table record ' do
 
-    database "Customer" do
+    database 'Customer' do
       strategy DataAnon::Strategy::Blacklist
       source_db connection_spec
 
@@ -27,5 +27,49 @@ describe "End 2 End RDBMS Blacklist Acceptance Test using SQLite database" do
     new_rec = source.find(CustomerSample::SAMPLE_DATA[:cust_id])
     new_rec['email'].should == 'test+1@gmail.com'
 
+  end
+
+  it 'should skip anonymization of the record if condition in skip is true' do
+    database 'Customer' do
+      strategy DataAnon::Strategy::Blacklist
+      source_db connection_spec
+
+      table 'customers' do
+        skip { |index, record| record['age'] > 18 }
+
+        primary_key 'cust_id'
+        anonymize('email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+        anonymize 'terms_n_condition', 'age'
+      end
+    end
+
+    DataAnon::Utils::SourceDatabase.establish_connection connection_spec
+    source = DataAnon::Utils::SourceTable.create 'customers', ['cust_id']
+    new_rec = source.find(CustomerSample::SAMPLE_DATA[:cust_id])
+    new_rec['email'].should_not == 'test+1@gmail.com'
+
+  end
+
+  it 'should continue with anonymization of the record if condition in skip is true' do
+    database 'Customer' do
+      strategy DataAnon::Strategy::Blacklist
+      source_db connection_spec
+
+      table 'customers' do
+        continue { |index, record| record['age'] > 18 }
+
+        primary_key 'cust_id'
+        anonymize('email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+        anonymize 'terms_n_condition', 'age'
+      end
+    end
+
+    DataAnon::Utils::SourceDatabase.establish_connection connection_spec
+    source = DataAnon::Utils::SourceTable.create 'customers', ['cust_id']
+    new_rec = source.find(CustomerSample::SAMPLE_DATA[:cust_id])
+    new_rec['email'].should == 'test+1@gmail.com'
+
+
+
   end
 end

data/spec/support/customer_sample.rb

@@ -20,7 +20,7 @@ class CustomerSample
         t.float :latitude
         t.float :longitude
 
-        t.timestamps
+        t.timestamps null: true
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data-anonymization
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Sunit Parekh
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-09 00:00:00.000000000 Z
+date: 2015-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord