data-anonymization 0.8.0 → 0.8.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 1eed8d87b9e7060ee7d3813c77b52b838f789d92
-  data.tar.gz: bc475f130cf331debd4cbc835ab5391aa38c3129
+SHA256:
+  metadata.gz: 7c24dc45a72724bdc3a3d96977afbf04382ab3d40e04981b5b9a6676ffaf83f0
+  data.tar.gz: 1b9bfd7209781e2afc867248ad374e5cac923c975b1933691e0216f0684721c1
 SHA512:
-  metadata.gz: e8e6e315a8468c1208c3fd65cf3ea575bbce766290cbbfc9fe323a329a438a02abe63d3751dabda4dd7a1e39a147f0d75aaf88b95beb4c6f09cef7969730b09a
-  data.tar.gz: aef7a9d30b0d1868435445f91a1cbe6412c8628eabcaffa2e51ea74fa721d1a2f3a53aa54295aabcfee229ae7b9333065ab119056c40946dca39ffc78a520ed7
+  metadata.gz: 58d1c365443b98223f2a59f31eab5700ad8023df5924c61157104fb503946a84145d6d8b5d413f0549935644a83de9386dcd5e663dc7caa451ad13d20881d57e
+  data.tar.gz: 382c0e5c77ae689d09351969898baba8b5cc8dea3775293ce402fa4f953307141295ad76e8fbe589ebac7be3125a989083eb6e489087c050abb5c9d768518dc9

data/.gitignore

@@ -18,4 +18,5 @@ tmp
 .idea
 sample-data/chinook-empty.sqlite
 tmp
-examples/mongodb_whitelist_generated.rb
+examples/mongodb_whitelist_generated.rb
+data

data/.ruby-version

@@ -1 +1 @@
-ruby-2.3.1
+ruby-2.7.1

data/.travis.yml

@@ -1,8 +1,9 @@
 language: ruby
 services:
   - mongodb
-before_install: gem install bundler -v 1.12.5
+before_install: gem install bundler
 before_script: rake empty_dest
 rvm:
-  - 2.2.5
-  - 2.3.1
+  - 2.5.8
+  - 2.6.6
+  - 2.7.1

data/README.md

@@ -1,6 +1,17 @@
 # Data::Anonymization
 Afraid of using production data due to privacy issues? Data Anonymization is a tool that helps you build anonymized production data dumps which you can use for performance testing, security testing, debugging and development.
 
+## Java/Kotlin version
+
+Java/Kotlin version of tool supporting RDBMS databases is available with similar easy to use DSL. 
+* [Kotlin/Java Data Anonymization Tool](https://github.com/dataanon/data-anon)
+* [Kotlin Maven Sample Project](https://github.com/dataanon/dataanon-kotlin-sample)
+* [Java Maven Sample Project](https://github.com/dataanon/dataanon-java-sample)
+
+
+----------------------
+
+
 [<img src="https://secure.travis-ci.org/sunitparekh/data-anonymization.png?branch=master">](http://travis-ci.org/sunitparekh/data-anonymization)
 [<img src="https://gemnasium.com/sunitparekh/data-anonymization.png?travis">](https://gemnasium.com/sunitparekh/data-anonymization)
 [<img src="https://codeclimate.com/badge.png">](https://codeclimate.com/github/sunitparekh/data-anonymization)
@@ -70,22 +81,26 @@ Postgresql database having **composite primary key**
 
 ## Changelog
 
+#### 0.8.1 (Aug 19, 2017)
+1. Multi-threading support added by [stanislav-tyutin](https://github.com/stanislav-tyutin) using Pull Request.
+2. Fixed to work with Ruby 2.4.x, issue with Integer data type
+
 #### 0.8.0 (Oct 31, 2016)
-1. Upgraded to rails 5.x 
+1. Upgraded to rails 5.x
 
 #### 0.7.4 (Oct 29, 2016)
-1. Continue to work on rails 4.x. Minor changes based on feedback. 
+1. Continue to work on rails 4.x. Minor changes based on feedback.
 
 #### 0.8.0.rc1 (Sep 5, 2016)
-1. Upgraded to rails 5.0, please report any issue or use case not working. 
+1. Upgraded to rails 5.0, please report any issue or use case not working.
 
 #### 0.7.3 (Feb 5, 2016)
-1. Fixed issue with batchsize. Thanks to [Jan Raasch](https://github.com/janraasch) for sending pull request. 
+1. Fixed issue with batchsize. Thanks to [Jan Raasch](https://github.com/janraasch) for sending pull request.
 
 #### 0.7.2 (Sep 26, 2015)
 1. Upgraded MongoDB to latest gem version 2.1.0 and tested with MongoDB 3.x version.
-2. Upgraded gems to latest version 
-3. Adding limit functionality - Merge pull request #27 from yanismydj/master       
+2. Upgraded gems to latest version
+3. Adding limit functionality - Merge pull request #27 from yanismydj/master
 
 #### 0.7.1 (Jun 13, 2015)
 1. Fixed issues with empty array data for MongoDB
@@ -224,6 +239,7 @@ Read more about [blacklist and whitelist here](http://sunitspace.blogspot.in/201
 5. Make sure to give proper case for fields and table names.
 6. Use skip and continue to apply different strategies for records.
 7. Use 'limit' to limit the number of rows that will be imported in whitelist
+8. RDBMS databases utilizing schemas can be specified via `schema_search_path`: `source_db { ... schema_search_path: 'public,my_special_schema' }`
 
 ## DSL Generation
 
@@ -269,7 +285,7 @@ datanon generate_rdbms_dsl -a postgresql -h 123.456.7.8 -d production_db
 
 ```
 
-The relevant db gems must be installed so that AR has the adapters required to establish the connection to the databases. The script generates a file named **rdbms_whitelist_generated.rb** in the same location as the project.  
+The relevant db gems must be installed so that AR has the adapters required to establish the connection to the databases. The script generates a file named **rdbms_whitelist_generated.rb** in the same location as the project.
 
 ### MongoDB whitelist generation
 
@@ -374,7 +390,7 @@ has following attribute accessor
 </tr>
 <tr>
 <td align="left">Text</td>
-<td align="left"><a href="http://rubydoc.info/github/sunitparekh/data-anonymization/DataAnon/Strategy/Field/RandomUrl">RandomURL</a></td>
+<td align="left"><a href="http://rubydoc.info/github/sunitparekh/data-anonymization/DataAnon/Strategy/Field/RandomUrl">RandomUrl</a></td>
 <td align="left">Anonymizes a URL while mainting the structure</td>
 </tr>
 </table><table>

data/commands.txt

@@ -0,0 +1,4 @@
+bundle update
+bundle outdated
+
+bundle exec rake

data/data-anonymization.gemspec

@@ -19,12 +19,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
 
-  gem.add_dependency('activerecord', '~> 5.0')
-  gem.add_dependency('composite_primary_keys', '~> 9.0')
-  gem.add_dependency('activesupport', '~> 5.0')
-  gem.add_dependency('rgeo', '~> 0.5')
-  gem.add_dependency('rgeo-geojson', '~> 0.4')
+  gem.add_dependency('activerecord', '~> 6.0')
+  gem.add_dependency('composite_primary_keys', '~> 12.0')
+  gem.add_dependency('activesupport', '~> 6.0')
+  gem.add_dependency('rgeo', '~> 1.0')
+  gem.add_dependency('rgeo-geojson', '~> 2.0')
   gem.add_dependency('powerbar', '~> 1.0')
-  gem.add_dependency('parallel', '~> 1.9')
-  gem.add_dependency('thor', '~> 0.19')
+  gem.add_dependency('parallel', '~> 1.12')
+  gem.add_dependency('thor', '~> 0.20.3')
 end

data/examples/blacklist_dsl.rb

@@ -1,4 +1,4 @@
-system "bundle exec ruby examples/whitelist_dsl.rb"
+system 'bundle exec ruby examples/whitelist_dsl.rb'
 
 require 'data-anonymization'
 
@@ -18,7 +18,7 @@ database 'Chinook' do
     anonymize('City').using FieldStrategy::RandomCity.region_US
     anonymize('State').using FieldStrategy::RandomProvince.region_US
     anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
-    anonymize('Country') {|field| "USA" }
+    anonymize('Country') {|field| 'USA'}
     anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
@@ -33,7 +33,7 @@ database 'Chinook' do
     anonymize('City').using FieldStrategy::RandomCity.region_US
     anonymize('State').using FieldStrategy::RandomProvince.region_US
     anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
-    anonymize('Country') {|field| "USA" }
+    anonymize('Country') {|field| 'USA'}
     anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
   end

data/examples/mongodb_blacklist_dsl.rb

@@ -4,20 +4,20 @@ require 'mongo'
 DataAnon::Utils::Logging.logger.level = Logger::INFO
 Mongo::Logger.logger.level = Logger::WARN
 
-Mongo::Client.new("mongodb://localhost/test").database.drop
-system "mongoimport --host=127.0.0.1 -d test --drop -c users --jsonArray ./sample-data/mongo/users.json"
-system "mongoimport --host=127.0.0.1 -d test --drop -c plans --jsonArray ./sample-data/mongo/plans.json"
+Mongo::Client.new('mongodb://localhost/test').database.drop
+system 'mongoimport --host=127.0.0.1 -d test --drop -c users --jsonArray ./sample-data/mongo/users.json'
+system 'mongoimport --host=127.0.0.1 -d test --drop -c plans --jsonArray ./sample-data/mongo/plans.json'
 
 
 database 'test' do
   strategy DataAnon::Strategy::MongoDB::Blacklist
-  source_db :mongodb_uri => "mongodb://localhost/test", :database => 'test'
+  source_db :mongodb_uri => 'mongodb://localhost/test', :database => 'test'
 
   collection 'users' do
     anonymize('date_of_birth').using FieldStrategy::TimeDelta.new(5,30)
     anonymize('user_id').using FieldStrategy::StringTemplate.new('user-#{row_number}')
     anonymize('email').using FieldStrategy::RandomMailinatorEmail.new
-    anonymize('password') { |field| "password" }
+    anonymize('password') { |field| 'password'}
     anonymize('first_name').using FieldStrategy::RandomFirstName.new
     anonymize('last_name').using FieldStrategy::RandomLastName.new
   end

data/examples/mongodb_whitelist_dsl.rb

@@ -5,22 +5,22 @@ DataAnon::Utils::Logging.logger.level = Logger::INFO
 Mongo::Logger.logger.level = Logger::WARN
 
 
-Mongo::Client.new("mongodb://localhost/test").database.drop
-Mongo::Client.new("mongodb://localhost/dest").database.drop
-system "mongoimport --host=127.0.0.1 -d test --drop -c users --jsonArray ./sample-data/mongo/users.json"
-system "mongoimport --host=127.0.0.1 -d test --drop -c plans --jsonArray ./sample-data/mongo/plans.json"
+Mongo::Client.new('mongodb://localhost/test').database.drop
+Mongo::Client.new('mongodb://localhost/dest').database.drop
+system 'mongoimport --host=127.0.0.1 -d test --drop -c users --jsonArray ./sample-data/mongo/users.json'
+system 'mongoimport --host=127.0.0.1 -d test --drop -c plans --jsonArray ./sample-data/mongo/plans.json'
 
 database 'test' do
   strategy DataAnon::Strategy::MongoDB::Whitelist
-  source_db :mongodb_uri => "mongodb://localhost/test", :database => 'test'
-  destination_db :mongodb_uri => "mongodb://localhost/dest", :database => 'dest'
+  source_db :mongodb_uri => 'mongodb://localhost/test', :database => 'test'
+  destination_db :mongodb_uri => 'mongodb://localhost/dest', :database => 'dest'
 
   collection 'users' do
     whitelist '_id','failed_attempts','updated_at'
     anonymize('date_of_birth').using FieldStrategy::TimeDelta.new(5,30)
     anonymize('user_id').using FieldStrategy::StringTemplate.new('user-#{row_number}')
     anonymize('email').using FieldStrategy::RandomMailinatorEmail.new
-    anonymize('password') { |field| "password" }
+    anonymize('password') { |field| 'password'}
     anonymize('first_name').using FieldStrategy::RandomFirstName.new
     anonymize('last_name').using FieldStrategy::RandomLastName.new
     anonymize 'password_reset_answer','password_reset_question'

data/examples/whitelist_dsl.rb

@@ -1,4 +1,4 @@
-system "rake empty_dest" # clean destination database on every call
+system 'rake empty_dest' # clean destination database on every call
 
 require 'data-anonymization'
 
@@ -15,7 +15,7 @@ database 'Chinook' do
     primary_key 'GenreId'
     whitelist 'GenreId'
     anonymize 'Name' do |field|
-      field.value + " test"
+      field.value + ' test'
     end
   end
 
@@ -39,7 +39,7 @@ database 'Chinook' do
     anonymize('City').using FieldStrategy::RandomCity.region_US
     anonymize('State').using FieldStrategy::RandomProvince.region_US
     anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
-    anonymize('Country') {|field| "USA" }
+    anonymize('Country') {|field| 'USA'}
     anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
   end
@@ -56,7 +56,7 @@ database 'Chinook' do
     anonymize('City').using FieldStrategy::RandomCity.region_US
     anonymize('State').using FieldStrategy::RandomProvince.region_US
     anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
-    anonymize('Country') {|field| "USA" }
+    anonymize('Country') {|field| 'USA'}
     anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')

data/examples/whitelist_dsl_threads.rb

@@ -0,0 +1,66 @@
+system 'rake empty_dest' # clean destination database on every call
+
+require 'data-anonymization'
+
+DataAnon::Utils::Logging.logger.level = Logger::INFO
+
+database 'Chinook' do
+  strategy DataAnon::Strategy::Whitelist
+  source_db :adapter => 'sqlite3', :database => 'sample-data/chinook.sqlite'
+  destination_db :adapter => 'sqlite3', :database => 'sample-data/chinook-empty.sqlite'
+
+  default_field_strategies :string => FieldStrategy::StringTemplate.new('Sunit #{row_number} Parekh')
+
+  table 'Genre' do
+    primary_key 'GenreId'
+    whitelist 'GenreId'
+    anonymize 'Name' do |field|
+      field.value + ' test'
+    end
+  end
+
+  table 'MediaType' do
+    primary_key 'MediaTypeId'
+    anonymize('MediaTypeId') { |field| field.value } # same as whitelist
+    anonymize('Name').using FieldStrategy::StringTemplate.new('Media Type #{row_number}')
+
+  end
+
+  table 'Customer' do
+    primary_key 'CustomerId'
+    thread_num 5  # thread_num
+
+    whitelist 'CustomerId', 'SupportRepId', 'Company'
+    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('FirstName').using FieldStrategy::RandomFirstName.new
+    anonymize('LastName').using FieldStrategy::RandomLastName.new
+    anonymize('Address').using FieldStrategy::RandomAddress.region_US
+    anonymize('City').using FieldStrategy::RandomCity.region_US
+    anonymize('State').using FieldStrategy::RandomProvince.region_US
+    anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
+    anonymize('Country') {|field| 'USA'}
+    anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+  end
+
+  table 'Employee' do
+    thread_num 5  # thread_num
+
+    whitelist 'EmployeeId', 'ReportsTo', 'Title'
+    anonymize('BirthDate').using FieldStrategy::DateTimeDelta.new(1, 1)
+    anonymize('FirstName').using FieldStrategy::RandomFirstName.new
+    anonymize('LastName').using FieldStrategy::RandomLastName.new
+    anonymize('HireDate').using FieldStrategy::DateTimeDelta.new(2, 0)
+    anonymize('Address').using FieldStrategy::RandomAddress.region_US
+    anonymize('City').using FieldStrategy::RandomCity.region_US
+    anonymize('State').using FieldStrategy::RandomProvince.region_US
+    anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
+    anonymize('Country') {|field| 'USA'}
+    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+  end
+
+
+
+end

data/lib/core/field.rb

@@ -3,7 +3,7 @@ module DataAnon
 
     class Field
 
-      def initialize name, value, row_number, ar_record, table_name = "unknown"
+      def initialize name, value, row_number, ar_record, table_name = 'unknown'
         @name = name
         @value = value
         @row_number = row_number

data/lib/core/table_errors.rb

@@ -11,7 +11,7 @@ module DataAnon
 
       def log_error record, exception
         @errors << { :record => record, :exception => exception}
-        raise "Reached limit of error for a table" if @errors.length > 100
+        raise 'Reached limit of error for a table' if @errors.length > 100
       end
 
       def errors

data/lib/data-anonymization.rb

@@ -1,28 +1,28 @@
-require "version"
+require 'version'
 
-require "utils/logging"
-require "utils/random_int"
-require "utils/random_float"
-require "utils/random_string"
-require "utils/random_string_chars_only"
-require "utils/geojson_parser"
-require "utils/progress_bar"
-require "utils/parallel_progress_bar"
-require "utils/resource"
-require "utils/template_helper"
-require "parallel/table"
-require "core/database"
-require "core/fields_missing_strategy"
-require "thor/helpers/rdbms_dsl_generator"
-require "core/field"
-require "core/table_errors"
-require "strategy/strategies"
-require "utils/database"
-require "core/dsl"
+require 'utils/logging'
+require 'utils/random_int'
+require 'utils/random_float'
+require 'utils/random_string'
+require 'utils/random_string_chars_only'
+require 'utils/geojson_parser'
+require 'utils/progress_bar'
+require 'utils/parallel_progress_bar'
+require 'utils/resource'
+require 'utils/template_helper'
+require 'parallel/table'
+require 'core/database'
+require 'core/fields_missing_strategy'
+require 'thor/helpers/rdbms_dsl_generator'
+require 'core/field'
+require 'core/table_errors'
+require 'strategy/strategies'
+require 'utils/database'
+require 'core/dsl'
 
 begin
   require 'mongo'
-  require "thor/helpers/mongodb_dsl_generator"
+  require 'thor/helpers/mongodb_dsl_generator'
 rescue LoadError
-  "Ignoring the mongodb specific libraries if monog driver is not specified in gem"
+  'Ignoring the mongodb specific libraries if monog driver is not specified in gem'
 end

data/lib/strategy/base.rb

@@ -37,6 +37,10 @@ module DataAnon
         @limit = limit
       end
 
+      def thread_num thread_num
+        @thread_num = thread_num
+      end
+
       def whitelist *fields
         fields.each { |f| @fields[f] = DataAnon::Strategy::Field::Whitelist.new }
       end
@@ -95,13 +99,15 @@ module DataAnon
           progress = progress_bar.new(@name, total)
           if @primary_keys.empty? || !@batch_size.present?
             process_table progress
+          elsif @thread_num.present?
+            process_table_in_threads progress
           else
             process_table_in_batches progress
           end
           progress.close
         end
         if source_table.respond_to?('clear_all_connections!')
-	        source_table.clear_all_connections!
+          source_table.clear_all_connections!
         end
       end
 
@@ -134,6 +140,40 @@ module DataAnon
         end
       end
 
+      def process_table_in_threads progress
+        logger.info "Processing table #{@name} records in batch size of #{@batch_size} [THREADS]"
+
+        index = 0
+        threads = []
+
+        source_table.find_in_batches(batch_size: @batch_size) do |records|
+          until threads.count(&:alive?) <= @thread_num
+            thr = threads.delete_at 0
+            thr.join
+            progress.show index
+          end
+
+          thr = Thread.new {
+            records.each do |record|
+              begin
+                process_record_if index, record
+                index += 1
+              rescue => exception
+                puts exception.inspect
+                @errors.log_error record, exception
+              end
+            end
+          }
+          threads << thr
+        end
+
+        until threads.empty?
+          thr = threads.delete_at 0
+          thr.join
+          progress.show index
+        end
+      end
+
       def source_table_limited
         @source_table_limited ||= begin
           if @limit.present?