data-anonymization 0.1.2

data/.documentup.json

@@ -0,0 +1,8 @@
+{
+    "repo":  "sunitparekh/data-anonymization",
+    "name": "Data Anonymization",
+    "theme": "v1",
+    "travis": true,
+    "twitter": ["dataanon"],
+    "google_analytics":"UA-34000799-1"
+}

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea
+sample-data/chinook-empty.sqlite
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+#--profile

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.9.3-p125@data-anon --create

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+before_install: gem install bundler --pre
+before_script: rake empty_dest
+rvm:
+  - 1.9.2
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development, :test do
+  gem 'foreman'
+  gem 'rake'
+  gem 'rspec'
+  gem 'pry'
+  gem 'sqlite3'
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Sunit Parekh
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,256 @@
+# Data::Anonymization
+Tool to create anonymized production data dump to use for PREF and other TEST environments.
+
+## Getting started
+Install gem using:
+
+    $ gem install data-anonymization
+
+Create ruby program using data-anonymization DSL as following `my_dsl.rb`:
+
+```ruby
+require 'data-anonymization'
+DF = DataAnon::Strategy::Field
+
+database 'DatabaseName' do
+  strategy DataAnon::Strategy::Blacklist  # whitelist (default) or blacklist
+
+  # database config as active record connection hash
+  source_db :adapter => 'sqlite3', :database => 'sample-data/chinook-empty.sqlite'
+
+  table 'User' do
+    primary_key 'id'
+    anonymize 'DateOfBirth' # uses default anonymization based on data types
+    anonymize('UserName').using DF::StringTemplate.new('user#{row_number}')
+    anonymize('Password') { |field| "password" }
+  end
+
+  ...
+
+end
+```
+
+Run using:
+
+    $ ruby my_dsl.rb
+
+### Share feedback
+Please use Github [issues](https://github.com/sunitparekh/data-anonymization/issues) to share feedback, feature suggestions or found any issues.
+
+Read more to learn all the features of the tool...
+
+## What is data anonymization?
+
+For almost all the project it is almost a need to have production data dump to run performance tests, rehearsal production releases and debugging production issues.
+However, getting production data and using it is not feasible due to multiple reasons and one of them is users personal data in database. And hence the need of data anonymization.
+This tool helps you to get anonymized production data dump using either Blacklist or Whitelist strategies.
+
+## Anonymization Strategies
+
+### Blacklist
+This approach is essentially to leave all fields unchanged with the exception of a few which are scrambled/anonymized (hence the name blacklist).
+Blacklist create a copy of prod database and choose the fields to be anonymized like e.g. username, password, email, name, geo location etc. Most of the fields had different rules e.g. password as always set to same value for all user, email need to be valid email (we used gmail trick with +N appended to it).
+Problem with this approach is, if new fields are added it will not be anonymized be default. Risk of user personal data passing through in future.
+
+```ruby
+database 'DatabaseName' do
+  strategy DataAnon::Strategy::Blacklist
+  source_db :adapter => 'sqlite3', :database => 'sample-data/chinook-empty.sqlite'
+  ...
+end
+```
+
+### Whitelist
+This approach is essentially to scramble/anonymize all fields except list of fields which are allowed to copy called as whitelist.
+By default all data needs to be anonymized. So from production database sanitizing the data record by record and insert anonymized data into destination database. Source database is kind of readonly.
+Have default anonymization rules based on data types. Have special rules for fields like username, password, email, name, geo location etc. And have list of whitelist fields means its okay to copy the data and doesn't need anonymization.
+This way any new field will be default get anonymized and if we need them as is, add it to the whitelist explicitly.
+
+```ruby
+database 'DatabaseName' do
+  strategy DataAnon::Strategy::Whitelist
+  source_db :adapter => 'sqlite3', :database => 'sample-data/chinook.sqlite'
+  destination_db :adapter => 'sqlite3', :database => 'sample-data/chinook-empty.sqlite'
+  ...
+end
+```
+
+## DataAnon::Core::Field
+The object that gets passed along with the field strategies.
+
+has following attribute accessor
+
+- `name` current field/column name
+- `value` current field/column value
+- `row_number` current row number
+- `ar_record` active record of the current row under processing
+
+## Field Strategies
+
+### LoremIpsum
+Default anonymization strategy for `string` content. Uses default 'Lorem ipsum...' text or text supplied in strategy to generate same length string.
+
+```ruby
+anonymize('UserName').using DataAnon::Strategy::Field::LoremIpsum.new
+```
+```ruby
+anonymize('UserName').using DataAnon::Strategy::Field::LoremIpsum.new("very large string....")
+```
+```ruby
+anonymize('UserName').using DataAnon::Strategy::Field::LoremIpsum.new(File.read('my_file.txt'))
+```
+
+### RandomString
+Generates random string of same length.
+```ruby
+anonymize('UserName').using DataAnon::Strategy::Field::RandomString.new
+```
+
+### StringTemplate
+Simple string evaluation within [DataAnon::Core::Field](#dataanon-core-field) context. Can be used for email, username anonymization.
+Make sure to put the string in 'single quote' else it will get evaluated inline.
+```ruby
+anonymize('UserName').using DataAnon::Strategy::Field::StringTemplate.new('user#{row_number}')
+```
+```ruby
+anonymize('Email').using DataAnon::Strategy::Field::StringTemplate.new('valid.address+#{row_number}@gmail.com')
+```
+```ruby
+anonymize('Email').using DataAnon::Strategy::Field::StringTemplate.new('useremail#{row_number}@mailinator.com')
+```
+
+### DateTimeDelta
+Shifts data randomly within given range. Default shifts date within 10 days + or - and shifts time within 30 minutes.
+```ruby
+anonymize('DateOfBirth').using DataAnon::Strategy::Field::DateTimeDelta.new
+```
+```ruby
+# shifts date within 20 days and time within 50 minutes
+anonymize('DateOfBirth').using DataAnon::Strategy::Field::DateTimeDelta.new(20, 50)
+```
+
+### RandomEmail
+Generates email randomly using the given HOSTNAME and TLD.
+By defaults generates hostname randomly along with email id.
+```ruby
+anonymize('DateOfBirth').using DataAnon::Strategy::Field::RandomEmail.new('thoughtworks','com')
+```
+
+### RandomMailinatorEmail
+Generates random email using mailinator hostname. e.g. <randomstring>@mailinator.com
+```ruby
+anonymize('DateOfBirth').using DataAnon::Strategy::Field::RandomMailinatorEmail.new
+```
+
+### RandomUserName
+### RandomFirstName
+### RandomLastName
+### RandomFullName
+### RandomInt
+### RandomIntegerDelta
+### RandomFloatDelta
+
+- - -
+
+
+
+- - -
+
+## Write you own field strategies
+field parameter in following code is [DataAnon::Core::Field](#dataanon-core-field)
+
+```ruby
+class MyFieldStrategy
+
+    # method anonymize is what required
+    def anonymize field
+      # write your code here
+    end
+
+end
+```
+
+write your own anonymous field strategies within DSL,
+
+```ruby
+  table 'User' do
+    anonymize('Password') { |field| "password" }
+    anonymize('email') do |field|
+        "test+#{field.row_number}@gmail.com"
+    end
+  end
+```
+
+
+## Default field strategies
+
+```ruby
+# Work in progress...
+DEFAULT_STRATEGIES = {:string => FS::LoremIpsum.new,
+                      :integer => FS::RandomInt.new(18,70),
+                      :datetime => FS::DateTimeDelta.new,
+                      :boolean => FS::RandomBoolean.new
+}
+```
+
+Overriding default field strategies,
+
+```ruby
+database 'Chinook' do
+  ...
+  default_field_strategies  :string => DataAnon::Strategy::Field::RandomString.new
+  ...
+end
+```
+
+## Examples
+
+1. [Whitelist](https://github.com/sunitparekh/data-anonymization/blob/master/whitelist_dsl.rb)
+2. [Blacklist](https://github.com/sunitparekh/data-anonymization/blob/master/blacklist_dsl.rb)
+
+
+## Logging
+
+`Progress Logger` provides progress of anonymization execution table by table.
+
+```ruby
+DataAnon::Utils::Logging.progress_logger.level = Logger::WARN
+```
+
+`Logger` provides debug level messages including database queries of active record.
+
+```ruby
+DataAnon::Utils::Logging.logger.level = Logger::INFO
+```
+
+## Changelog
+
+
+### 0.1.1 (August 13, 2012)
+
+1. First initial release
+
+## What's plan ahead?
+
+1. Run anonymization in parallel threads (performance enchantments)
+2. MongoDB anonymization support (NoSQL document based database support)
+
+## Want to contribute?
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## License
+
+[MIT License](https://github.com/sunitparekh/data-anonymization/blob/master/LICENSE.txt)
+
+## Credits
+
+- [ThoughtWorks Inc](http://www.thoughtworks.com), for allowing us to build this tool and make if open source.
+- [Birinder](https://twitter.com/birinder_) and [Panda](https://twitter.com/sarbashrestha) for reviewing the documentation.
+
+
+

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler'
+require 'rspec/core/rake_task'
+require "tasks/rake_tasks"
+
+Bundler::GemHelper.install_tasks
+RSpec::Core::RakeTask.new(:spec)
+DataAnonymization::RakeTasks.new
+
+task :default => :spec

data/blacklist_dsl.rb

@@ -0,0 +1,19 @@
+system "bundle exec ruby whitelist_dsl.rb"
+
+require 'data-anonymization'
+
+FS = DataAnon::Strategy::Field
+
+DataAnon::Utils::Logging.logger.level = Logger::INFO
+
+database 'Chinook' do
+  strategy DataAnon::Strategy::Blacklist
+  source_db :adapter => 'sqlite3', :database => 'sample-data/chinook-empty.sqlite'
+
+  table 'MediaType' do
+    primary_key 'MediaTypeId'
+    anonymize('Name').using FS::StringTemplate.new('Media Type 100#{row_number}')
+  end
+
+end
+

data/blacklist_nosql_dsl.rb

@@ -0,0 +1,36 @@
+require 'data-anonymization'
+
+FS = DataAnon::Strategy::Field
+
+DataAnon::Utils::Logging.logger.level = Logger::INFO
+
+# DSL for NOSQL database, NOT IMPLEMENTED YET
+
+database 'Chinook' do
+  strategy DataAnon::Strategy::NoSQL::Blacklist
+  source_db :adapter => 'sqlite3', :database => 'sample-data/chinook-empty.sqlite'
+
+  document 'User' do
+    primary_key 'MediaTypeId'
+    whitelist
+
+    node 'address' do
+      whitelist 'addrssline1', 'addressline2'
+      anonymize
+      anonymize 'pincode'
+
+      node 'contacts' do
+        anonymize('phone-number').using FS::RandomPhoneNumber.new
+      end
+
+      anonymize 'contacts/phone-number'
+
+    end
+
+    anonymize 'address/contacts/phone-number'
+
+    anonymize('Name').using FS::StringTemplate.new('Media Type 100#{row_number}')
+  end
+
+end
+

data/data-anonymization.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "data-anonymization"
+  gem.version       = DataAnonymization::VERSION
+  gem.authors       = ["Sunit Parekh", "Anand Agrawal", "Satyam Agarwala"]
+  gem.email         = ["parekh.sunit@gmail.com","anand.agrawal84@gmail.com", "satyamag@gmail.com"]
+  gem.description   = %q{Data anonymization tool for RDBMS databases}
+  gem.summary       = %q{Tool to create anonymized production data dump to use for PREF and other TEST environments.}
+  gem.homepage      = "http://sunitparekh.github.com/data-anonymization"
+
+  gem.files         = `git ls-files`.split($/).select { |f| !f.match(/^sample-data/) }
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency('activerecord', '~> 3.2.8')
+  gem.add_dependency('activesupport', '~> 3.2.8')
+end

data/lib/core/database.rb

@@ -0,0 +1,36 @@
+module DataAnon
+  module Core
+
+    class Database
+
+      def initialize name
+        @name = name
+        @strategy = DataAnon::Strategy::Whitelist
+        @user_defaults = {}
+      end
+
+      def strategy strategy
+        @strategy = strategy
+      end
+
+      def source_db connection_spec
+        DataAnon::Utils::SourceDatabase.establish_connection connection_spec
+      end
+
+      def destination_db connection_spec
+        DataAnon::Utils::DestinationDatabase.establish_connection connection_spec
+      end
+
+      def default_field_strategies default_strategies
+        @user_defaults = default_strategies
+      end
+
+      def table (name, &block)
+        @strategy.new(name, @user_defaults).process_fields(&block).process
+      end
+
+
+    end
+
+  end
+end