data-anonymization 0.3.0 → 0.5.0

data/spec/core/fields_missing_strategy_spec.rb

@@ -0,0 +1,26 @@
+require "spec_helper"
+
+describe DataAnon::Core::FieldsMissingStrategy do
+
+  FMS = DataAnon::Core::FieldsMissingStrategy
+
+  it "should be able to add field for new table that doesn't exist" do
+    users = FMS.new("users")
+    users.missing("confirm_email")
+    users.fields_missing_strategy.should == ["confirm_email"]
+  end
+
+  it "should be able to take care for same field appearing multiple time" do
+    users = FMS.new("users")
+    users.missing("confirm_email")
+    users.missing("confirm_email")
+    users.fields_missing_strategy.should == ["confirm_email"]
+  end
+
+  it "should be able to add multiple fields for table" do
+    users = FMS.new("users")
+    users.missing("confirm_email")
+    users.missing("password_reset")
+    users.fields_missing_strategy.should == ["confirm_email","password_reset"]
+  end
+end

data/spec/strategy/field/name/random_first_name_spec.rb

@@ -6,7 +6,7 @@ describe FieldStrategy::RandomFirstName do
   let(:field) {DataAnon::Core::Field.new('firstname','fakeFirstName',1,nil)}
 
   describe 'anonymized name must not be the same as provided name' do
-    let(:anonymized_value) {RandomFirstName.new().anonymize(field.value)}
+    let(:anonymized_value) {RandomFirstName.new().anonymize(field)}
 
     it {anonymized_value.should_not equal field.value}
   end

data/spec/strategy/field/name/random_full_name_spec.rb

@@ -3,21 +3,26 @@ require "spec_helper"
 describe FieldStrategy::RandomFullName do
 
   RandomFullName = FieldStrategy::RandomFullName
-  let(:field) {DataAnon::Core::Field.new('name','Fake User',1,nil)}
 
-  describe 'anonymized name should be the same as original' do
+  describe 'anonymized name with just single name' do
+    let(:field) {DataAnon::Core::Field.new('name','Fake',1,nil)}
+    let(:anonymized_value) {RandomFullName.new().anonymize(field)}
 
-    let(:anonymized_name) {RandomFullName.new().anonymize(field)}
+    it {anonymized_value.should_not equal field.value}
+  end
 
-    it {anonymized_name.should_not equal field.value}
+  describe 'anonymized name should be the same as original' do
+    let(:field) {DataAnon::Core::Field.new('name','Fake User',1,nil)}
+    let(:anonymized_value) {RandomFullName.new().anonymize(field)}
+
+    it {anonymized_value.should_not equal field.value}
   end
 
   describe 'anonymized name should have same number of words as original' do
-
     let(:field) {DataAnon::Core::Field.new('name','Fake User Longer Name Test',1,nil)}
-    let(:anonymized_name) {RandomFullName.new().anonymize(field)}
+    let(:anonymized_value) {RandomFullName.new().anonymize(field)}
 
-    it {anonymized_name.split(' ').size.should equal field.value.split(' ').size}
+    it {anonymized_value.split(' ').size.should equal 5}
   end
 
 end

data/spec/strategy/field/name/random_last_name_spec.rb

@@ -6,7 +6,7 @@ describe FieldStrategy::RandomLastName do
   let(:field) {DataAnon::Core::Field.new('lastname','fakeLastName',1,nil)}
 
   describe 'anonymized name must not be the same as provided name' do
-    let(:anonymized_value) {RandomLastName.new().anonymize(field.value)}
+    let(:anonymized_value) {RandomLastName.new().anonymize(field)}
 
     it {anonymized_value.should_not equal field.value}
   end

data/spec/strategy/field/string/random_formatted_string_spec.rb

@@ -0,0 +1,39 @@
+require "spec_helper"
+
+describe FieldStrategy::RandomFormattedString do
+
+  RandomFormattedString = FieldStrategy::RandomFormattedString
+
+  describe 'anonymized credit card number preserving the format' do
+    let(:field) {DataAnon::Core::Field.new('credit_card_number',"1111-2222-3333-4444",1,nil)}
+    let(:anonymized_value) {RandomFormattedString.new.anonymize(field)}
+
+    it {anonymized_value.should_not equal field.value}
+    it { anonymized_value.should match /^\d{4}-\d{4}-\d{4}-\d{4}$/}
+  end
+
+  describe 'anonymized email preserving the format' do
+    let(:field) {DataAnon::Core::Field.new('email',"parekh1.sunit@gmail.com",1,nil)}
+    let(:anonymized_value) {RandomFormattedString.new.anonymize(field)}
+
+    it {anonymized_value.should_not equal field.value}
+    it { anonymized_value.should match /^[a-z]{6}\d\.[a-z]{5}@[a-z]{5}\.[a-z]{3}$/}
+  end
+
+  describe 'anonymized string preserving the string case & format' do
+    let(:field) {DataAnon::Core::Field.new('email',"parekh1.SUNIT@gmail.com",1,nil)}
+    let(:anonymized_value) {RandomFormattedString.new.anonymize(field)}
+
+    it {anonymized_value.should_not equal field.value}
+    it { anonymized_value.should match /^[a-z]{6}\d\.[A-Z]{5}@[a-z]{5}\.[a-z]{3}$/}
+  end
+
+  describe 'anonymized phone# preserving the format' do
+    let(:field) {DataAnon::Core::Field.new('home_phone',"(020)3423-8013",1,nil)}
+    let(:anonymized_value) {RandomFormattedString.new.anonymize(field)}
+
+    it {anonymized_value.should_not equal field.value}
+    it { anonymized_value.should match /^\(\d{3}\)\d{4}-\d{4}$/}
+  end
+
+end

data/spec/strategy/field/string/select_from_file_spec.rb

@@ -0,0 +1,21 @@
+require "spec_helper"
+
+describe FieldStrategy::SelectFromFile do
+
+  SelectFromFile = FieldStrategy::SelectFromFile
+
+  describe 'anonymized name must not be the same as provided name' do
+    let(:field) {DataAnon::Core::Field.new('firstname','fakeFirstName',1,nil)}
+    let(:anonymized_value) {SelectFromFile.new(DataAnon::Utils::Resource.file('first_names.txt')).anonymize(field)}
+
+    it {anonymized_value.should_not equal field.value}
+  end
+
+  describe 'anonymized multiple values' do
+    let(:field) {DataAnon::Core::Field.new('firstname',['value1','value2'],1,nil)}
+    let(:anonymized_values) {SelectFromFile.new(DataAnon::Utils::Resource.file('first_names.txt')).anonymize(field)}
+
+    it {anonymized_values.length.should equal 2}
+  end
+
+end

data/spec/strategy/mongodb/anonymize_field_spec.rb

@@ -0,0 +1,52 @@
+require "spec_helper"
+
+describe DataAnon::Strategy::MongoDB::AnonymizeField do
+
+
+  it 'should do callback recursive in case of sub document' do
+    sub_document = {'key' => 'value'}
+    field_strategy = {'key' => FieldStrategy::LoremIpsum.new}
+    anonymization_strategy = mock("AnonymizationStrategy")
+    anonymization_strategy.should_receive(:anonymize_document).with(sub_document,1,field_strategy).and_return({'key' => "anonymized_value"})
+    field = DataAnon::Core::Field.new('sub_document_field', sub_document,1,nil)
+    anonymize_field = DataAnon::Strategy::MongoDB::AnonymizeField.new(field, field_strategy,anonymization_strategy)
+    anonymized_value = anonymize_field.anonymize
+    anonymized_value['key'].should == "anonymized_value"
+  end
+
+  it 'should do callback recursive multiple time in case of array of sub document' do
+    sub_documents = [{'key' => 'value1'},{'key' => 'value2'}]
+    field_strategy = {'key' => FieldStrategy::LoremIpsum.new}
+    anonymization_strategy = mock("AnonymizationStrategy")
+    anonymization_strategy.should_receive(:anonymize_document).with({'key' => 'value1'},1,field_strategy).and_return({'key' => "anonymized_value1"})
+    anonymization_strategy.should_receive(:anonymize_document).with({'key' => 'value2'},1,field_strategy).and_return({'key' => "anonymized_value2"})
+    field = DataAnon::Core::Field.new('sub_document_field', sub_documents,1,nil)
+    anonymize_field = DataAnon::Strategy::MongoDB::AnonymizeField.new(field, field_strategy,anonymization_strategy)
+    anonymized_value = anonymize_field.anonymize
+    anonymized_value.length.should == 2
+    anonymized_value[0]['key'].should == "anonymized_value1"
+    anonymized_value[1]['key'].should == "anonymized_value2"
+  end
+
+  it 'should anonymize array field data type' do
+    anonymization_strategy = mock("AnonymizationStrategy")
+    anonymization_strategy.should_not_receive(:anonymize_document)
+    field = DataAnon::Core::Field.new('tags',['tag1','tag2'],1,nil)
+    anonymize_field = DataAnon::Strategy::MongoDB::AnonymizeField.new(field,FieldStrategy::SelectFromList.new(['tag4','tag5','tag6','tag7','tag8']),anonymization_strategy)
+    anonymized_value = anonymize_field.anonymize
+    anonymized_value.length == 2
+    ['tag4','tag5','tag6','tag7','tag8'].should include(anonymized_value[0])
+    ['tag4','tag5','tag6','tag7','tag8'].should include(anonymized_value[1])
+  end
+
+  it 'should anonymize field and return anonymized value using passed strategy' do
+    anonymization_strategy = mock("AnonymizationStrategy")
+    anonymization_strategy.should_not_receive(:anonymize_document)
+    field = DataAnon::Core::Field.new('boolean_field',false,1,nil)
+    anonymize_field = DataAnon::Strategy::MongoDB::AnonymizeField.new(field,FieldStrategy::RandomBoolean.new,anonymization_strategy)
+    anonymized_value = anonymize_field.anonymize
+    [true, false].should include(anonymized_value)
+  end
+
+
+end

data/spec/utils/random_float_spec.rb

@@ -0,0 +1,12 @@
+require "spec_helper"
+
+describe "Number Utils" do
+
+  describe 'should return same length value using default text' do
+
+    let(:random_float) { DataAnon::Utils::RandomFloat.generate(5,10) }
+
+    it { random_float.should be_between(5,10) }
+    it { random_float.should be_a_kind_of Float }
+  end
+end

data/spec/utils/random_string_char_only_spec.rb

@@ -0,0 +1,12 @@
+require "spec_helper"
+
+describe "String Utils" do
+
+  it "should generate random string of given length" do
+    DataAnon::Utils::RandomStringCharsOnly.generate(10).length.should equal 10
+  end
+
+  it "should generate random string only with characters" do
+    DataAnon::Utils::RandomStringCharsOnly.generate(10).should match /^[a-zA-Z]{10}$/
+  end
+end

data/spec/utils/template_helper_spec.rb

@@ -0,0 +1,14 @@
+require "spec_helper"
+
+describe "Template Helper" do
+
+  it "should return a correctly formatted string based on input connection hash for source" do
+    connection_hash = {:adapter => "test_adapter", :port => 5000}
+    DataAnon::Utils::TemplateHelper.source_connection_specs_rdbms(connection_hash).should eq(":adapter => 'test_adapter', :port => 5000")
+  end
+
+  it "should return a correctly formatted string based on input connection hash for destination" do
+    connection_hash = {:adapter => "test_adapter", :port => 5000}
+    DataAnon::Utils::TemplateHelper.destination_connection_specs_rdbms(connection_hash).should eq(":adapter => '<enter_value>', :port => '<enter_value>'")
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data-anonymization
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-04 00:00:00.000000000 Z
+date: 2012-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -125,12 +125,29 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.5.18
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.16.0
 description: Data anonymization tool for RDBMS databases
 email:
 - parekh.sunit@gmail.com
 - anand.agrawal84@gmail.com
 - satyamag@gmail.com
-executables: []
+executables:
+- datanon
 extensions: []
 extra_rdoc_files: []
 files:
@@ -143,16 +160,22 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- blacklist_dsl.rb
-- blacklist_nosql_dsl.rb
+- bin/datanon
 - data-anonymization.gemspec
+- examples/blacklist_dsl.rb
+- examples/mongodb_blacklist_dsl.rb
+- examples/mongodb_whitelist_dsl.rb
+- examples/whitelist_dsl.rb
 - lib/core/database.rb
 - lib/core/dsl.rb
 - lib/core/field.rb
+- lib/core/fields_missing_strategy.rb
+- lib/core/table_errors.rb
 - lib/data-anonymization.rb
 - lib/parallel/table.rb
 - lib/strategy/base.rb
 - lib/strategy/blacklist.rb
+- lib/strategy/field/anonymize_array.rb
 - lib/strategy/field/anonymous.rb
 - lib/strategy/field/contact/geojson_base.rb
 - lib/strategy/field/contact/random_address.rb
@@ -183,6 +206,7 @@ files:
 - lib/strategy/field/random_boolean.rb
 - lib/strategy/field/string/formatted_string_numbers.rb
 - lib/strategy/field/string/lorem_ipsum.rb
+- lib/strategy/field/string/random_formatted_string.rb
 - lib/strategy/field/string/random_string.rb
 - lib/strategy/field/string/random_url.rb
 - lib/strategy/field/string/select_from_database.rb
@@ -190,25 +214,37 @@ files:
 - lib/strategy/field/string/select_from_list.rb
 - lib/strategy/field/string/string_template.rb
 - lib/strategy/field/whitelist.rb
+- lib/strategy/mongodb/anonymize_field.rb
+- lib/strategy/mongodb/blacklist.rb
+- lib/strategy/mongodb/whitelist.rb
 - lib/strategy/strategies.rb
 - lib/strategy/whitelist.rb
 - lib/tasks/rake_tasks.rb
+- lib/thor/helpers/mongodb_dsl_generator.rb
+- lib/thor/helpers/rdbms_dsl_generator.rb
+- lib/thor/templates/mongodb_whitelist_template.erb
+- lib/thor/templates/whitelist_template.erb
 - lib/utils/database.rb
 - lib/utils/geojson_parser.rb
 - lib/utils/logging.rb
+- lib/utils/parallel_progress_bar.rb
 - lib/utils/progress_bar.rb
 - lib/utils/random_float.rb
 - lib/utils/random_int.rb
 - lib/utils/random_string.rb
 - lib/utils/random_string_chars_only.rb
 - lib/utils/resource.rb
+- lib/utils/template_helper.rb
 - lib/version.rb
 - resources/UK_addresses.geojson
 - resources/US_addresses.geojson
 - resources/first_names.txt
 - resources/last_names.txt
+- spec/acceptance/mongodb_blacklist_spec.rb
+- spec/acceptance/mongodb_whitelist_spec.rb
 - spec/acceptance/rdbms_blacklist_spec.rb
 - spec/acceptance/rdbms_whitelist_spec.rb
+- spec/core/fields_missing_strategy_spec.rb
 - spec/resource/sample.geojson
 - spec/spec_helper.rb
 - spec/strategy/field/contact/random_address_spec.rb
@@ -238,18 +274,23 @@ files:
 - spec/strategy/field/random_boolean_spec.rb
 - spec/strategy/field/string/formatted_string_numbers_spec.rb
 - spec/strategy/field/string/lorem_ipsum_spec.rb
+- spec/strategy/field/string/random_formatted_string_spec.rb
 - spec/strategy/field/string/random_string_spec.rb
 - spec/strategy/field/string/random_url_spec.rb
 - spec/strategy/field/string/select_from_database_spec.rb
+- spec/strategy/field/string/select_from_file_spec.rb
 - spec/strategy/field/string/select_from_list_spec.rb
 - spec/strategy/field/string/string_template_spec.rb
 - spec/strategy/field/whitelist_spec.rb
+- spec/strategy/mongodb/anonymize_field_spec.rb
 - spec/support/customer_sample.rb
 - spec/utils/database_spec.rb
 - spec/utils/geojson_parser_spec.rb
+- spec/utils/random_float_spec.rb
 - spec/utils/random_int_spec.rb
+- spec/utils/random_string_char_only_spec.rb
 - spec/utils/random_string_spec.rb
-- whitelist_dsl.rb
+- spec/utils/template_helper_spec.rb
 homepage: http://sunitparekh.github.com/data-anonymization
 licenses: []
 post_install_message: 
@@ -276,8 +317,11 @@ specification_version: 3
 summary: Tool to create anonymized production data dump to use for PREF and other
   TEST environments.
 test_files:
+- spec/acceptance/mongodb_blacklist_spec.rb
+- spec/acceptance/mongodb_whitelist_spec.rb
 - spec/acceptance/rdbms_blacklist_spec.rb
 - spec/acceptance/rdbms_whitelist_spec.rb
+- spec/core/fields_missing_strategy_spec.rb
 - spec/resource/sample.geojson
 - spec/spec_helper.rb
 - spec/strategy/field/contact/random_address_spec.rb
@@ -307,14 +351,20 @@ test_files:
 - spec/strategy/field/random_boolean_spec.rb
 - spec/strategy/field/string/formatted_string_numbers_spec.rb
 - spec/strategy/field/string/lorem_ipsum_spec.rb
+- spec/strategy/field/string/random_formatted_string_spec.rb
 - spec/strategy/field/string/random_string_spec.rb
 - spec/strategy/field/string/random_url_spec.rb
 - spec/strategy/field/string/select_from_database_spec.rb
+- spec/strategy/field/string/select_from_file_spec.rb
 - spec/strategy/field/string/select_from_list_spec.rb
 - spec/strategy/field/string/string_template_spec.rb
 - spec/strategy/field/whitelist_spec.rb
+- spec/strategy/mongodb/anonymize_field_spec.rb
 - spec/support/customer_sample.rb
 - spec/utils/database_spec.rb
 - spec/utils/geojson_parser_spec.rb
+- spec/utils/random_float_spec.rb
 - spec/utils/random_int_spec.rb
+- spec/utils/random_string_char_only_spec.rb
 - spec/utils/random_string_spec.rb
+- spec/utils/template_helper_spec.rb

data/blacklist_dsl.rb

@@ -1,17 +0,0 @@
-system "bundle exec ruby whitelist_dsl.rb"
-
-require 'data-anonymization'
-
-DataAnon::Utils::Logging.logger.level = Logger::INFO
-
-database 'Chinook' do
-  strategy DataAnon::Strategy::Blacklist
-  source_db :adapter => 'sqlite3', :database => 'sample-data/chinook-empty.sqlite'
-
-  table 'MediaType' do
-    primary_key 'MediaTypeId'
-    anonymize('Name').using FieldStrategy::StringTemplate.new('Media Type 100#{row_number}')
-  end
-
-end
-

data/blacklist_nosql_dsl.rb

@@ -1,36 +0,0 @@
-require 'data-anonymization'
-
-FS = DataAnon::Strategy::Field
-
-DataAnon::Utils::Logging.logger.level = Logger::INFO
-
-# DSL for NOSQL database, NOT IMPLEMENTED YET
-
-database 'Chinook' do
-  strategy DataAnon::Strategy::NoSQL::Blacklist
-  source_db :adapter => 'sqlite3', :database => 'sample-data/chinook-empty.sqlite'
-
-  document 'User' do
-    primary_key 'MediaTypeId'
-    whitelist
-
-    node 'address' do
-      whitelist 'addrssline1', 'addressline2'
-      anonymize
-      anonymize 'pincode'
-
-      node 'contacts' do
-        anonymize('phone-number').using FS::RandomPhoneNumber.new
-      end
-
-      anonymize 'contacts/phone-number'
-
-    end
-
-    anonymize 'address/contacts/phone-number'
-
-    anonymize('Name').using FS::StringTemplate.new('Media Type 100#{row_number}')
-  end
-
-end
-