data-anonymization 0.3.0 → 0.5.0

data/lib/thor/helpers/rdbms_dsl_generator.rb

@@ -0,0 +1,36 @@
+require 'thor'
+require 'active_record'
+require 'erb'
+
+module DataAnon
+  module ThorHelpers
+    class RDBMSDSLGenerator
+
+      def self.source_root
+        File.dirname(__FILE__)
+      end
+
+      def generate_whitelist_script(configuration_hash)
+
+        @configuration_hash = configuration_hash
+        @ar_object = ActiveRecord::Base.establish_connection(@configuration_hash)
+
+        @tables = @ar_object.connection.tables
+
+        erb = ERB.new( File.new(RDBMSDSLGenerator.source_root + "/../templates/whitelist_template.erb").read, nil, '-')
+
+        File.open('rdbms_whitelist_generated.rb', 'w') do |f|
+          f.write erb.result(binding)
+          f.close
+        end
+
+      rescue => e
+        puts "\e[31mActiverecord was unable to establish a connection to the specified database. Please check the configuration options and try again.\e[0m"
+        puts e.backtrace
+      end
+
+    end
+  end
+end
+
+

data/lib/thor/templates/mongodb_whitelist_template.erb

@@ -0,0 +1,15 @@
+require 'data-anonymization'
+require 'mongo'
+
+DataAnon::Utils::Logging.logger.level = Logger::INFO
+
+database 'test' do
+
+  strategy DataAnon::Strategy::MongoDB::Whitelist
+  source_db <%= DataAnon::Utils::TemplateHelper.source_connection_specs_mongo @configuration_hash %>
+  destination_db <%= DataAnon::Utils::TemplateHelper.destination_connection_specs_mongo %>
+
+<%= @output.join("\n")  %>
+
+end
+

data/lib/thor/templates/whitelist_template.erb

@@ -0,0 +1,21 @@
+require 'data-anonymization'
+
+DataAnon::Utils::Logging.logger.level = Logger::INFO
+
+database 'Template' do
+
+  strategy DataAnon::Strategy::Whitelist
+  source_db <%= DataAnon::Utils::TemplateHelper.source_connection_specs_rdbms @configuration_hash %>
+  destination_db <%= DataAnon::Utils::TemplateHelper.destination_connection_specs_rdbms @configuration_hash %>
+
+  <% @tables.each do |table| %>
+  table '<%= table %>' do
+    primary_key '<%= @ar_object.connection.primary_key("#{table}").nil? ? "<No primary key found. Possible composite key. Please enter the value>" : @ar_object.connection.primary_key("#{table}") %>'
+    <%- @ar_object.connection.indexes("#{table}").each do |index| -%>
+    whitelist '<%= index.columns.first %>'
+    <%- end -%>
+  end
+  <% end %>
+
+end
+

data/lib/utils/database.rb

@@ -14,6 +14,10 @@ module DataAnon
       self.abstract_class = true
     end
 
+    class DisableReferentialIntegrityDatabase < ActiveRecord::Base
+      self.abstract_class = true
+    end
+
     class SourceDatabase < ActiveRecord::Base
       self.abstract_class = true
     end

data/lib/utils/parallel_progress_bar.rb

@@ -0,0 +1,24 @@
+require 'powerbar'
+
+module DataAnon
+  module Utils
+
+    class ParallelProgressBar < ProgressBar
+      include Utils::Logging
+
+      def initialize table_name, total
+        @total = total
+        @table_name = table_name
+      end
+
+      protected
+
+      def show_progress index
+        suffix = started(index) ? "STARTED" : (complete(index) ? "COMPLETE" : "")
+        logger.info("%-30s [ %7d/%-7d ] %s" % [@table_name, index, @total, suffix])
+      end
+
+    end
+
+  end
+end

data/lib/utils/progress_bar.rb

@@ -4,51 +4,63 @@ module DataAnon
   module Utils
 
     class ProgressBar
-      include Utils::Logging
 
       def initialize table_name, total
         @total = total
         @table_name = table_name
-        @progress_bar = PowerBar.new if total > 0 && show_progress && !parallel?
+        @power_bar = PowerBar.new if show_progress_env
+        apply_power_bar_settings  if show_progress_env
       end
 
-      def show_progress
-        ENV['show_progress'] != 'false'
-      end
-
-      def parallel?
-        ENV['parallel_execution'] == 'true'
+      def apply_power_bar_settings
+        @power_bar.settings.tty.finite.template.main = \
+        "${<msg>} ${<bar> }\e[0m${<rate>/s} \e[33;1m${<percent>%} " +
+            "\e[36;1m${<elapsed>}\e[31;1m${ ETA: <eta>}"
+        @power_bar.settings.tty.finite.template.padchar = "\e[0m\u2589"
+        @power_bar.settings.tty.finite.template.barchar = "\e[34;1m\u2589"
+        @power_bar.settings.tty.finite.template.exit = "\e[?25h\e[0m" # clean up after us
+        @power_bar.settings.tty.finite.template.close = "\e[?25h\e[0m\n" # clean up after us
+        @power_bar.settings.tty.finite.output = Proc.new { |s| $stderr.print s }
       end
 
       def show index
-        if started(index) || regular_interval(index) || complete(index)
-          if @progress_bar
-            msg = "Table: %-15s [ %6d/%-6d ]" % [ @table_name,index,@total]
-            @progress_bar.show(:msg => msg, :done => index, :total => @total)
-          elsif parallel?
-            suffix = ""
-            suffix = "STARTED" if started(index)
-            suffix = "COMPLETE" if complete(index)
-            logger.info("Table: %-15s [ %6d/%-6d ] %s" % [ @table_name,index,@total, suffix])
-          end
+        if show_progress? index
+          show_progress index
         end
       end
 
+      def close
+        @power_bar.close if @power_bar
+      end
+
+      protected
+
+      def show_progress? index
+        show_progress_env && (started(index) || regular_interval(index) || complete(index))
+      end
+
+      def show_progress_env
+        ENV['show_progress'] == "false" ? false : true
+      end
+
+      def show_progress counter
+        sleep 0.1
+        msg = "%-20s [%6d/%-6d]" % [@table_name, counter, @total]
+        @power_bar.show({:msg => msg, :done => counter, :total => @total})
+      end
+
       def complete index
         index == @total
       end
 
       def regular_interval index
-        index % 1000 == 0
+        (index % 1000) == 0
       end
 
       def started index
         index == 1
       end
 
-      def close
-        @progress_bar.close if @progress_bar
-      end
 
     end
 

data/lib/utils/random_string.rb

@@ -2,9 +2,10 @@ module DataAnon
   module Utils
     class RandomString
 
-      def self.generate length = nil
+      RANDOM_STRING_CHARS = 'abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ0123456789'
+
+      def self.generate length = nil, chars = RANDOM_STRING_CHARS
         length ||= Random.new.rand 5...15
-        chars = 'abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789'
         random_string = ''
         length.times { random_string << chars[rand(chars.size)] }
         random_string

data/lib/utils/random_string_chars_only.rb

@@ -2,12 +2,10 @@ module DataAnon
   module Utils
     class RandomStringCharsOnly
 
+      CHARS = 'abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ'
+
       def self.generate length = nil
-        length ||= Random.new.rand 5...15
-        chars = 'abcdefghjkmnpqrstuvwxyz'
-        random_string = ''
-        length.times { random_string << chars[rand(chars.size)] }
-        random_string
+        RandomString.generate length, CHARS
       end
     end
   end

data/lib/utils/template_helper.rb

@@ -0,0 +1,44 @@
+module DataAnon
+  module Utils
+    class TemplateHelper
+
+      def self.source_connection_specs_rdbms config_hash
+
+        config_hash.keys.reject{|key| config_hash[key].nil? }.collect { |key|
+          if ((config_hash[key].class.to_s.downcase == "string"))
+            ":#{key} => '#{config_hash[key]}'"
+          elsif ((config_hash[key].class.to_s.downcase == "fixnum"))
+            ":#{key} => #{config_hash[key]}"
+          end
+        }.join ', '
+
+      end
+
+      def self.destination_connection_specs_rdbms config_hash
+
+        config_hash.keys.collect { |key|
+          ":#{key} => '<enter_value>'"
+        }.join ', '
+
+      end
+
+      def self.source_connection_specs_mongo config_hash
+        ":mongodb_uri => '#{self.mongo_uri config_hash}', :database => '#{config_hash[:database]}'"
+      end
+
+      def self.destination_connection_specs_mongo
+        ":mongodb_uri => '<enter value>', :database => '<enter value>'"
+      end
+
+      def self.mongo_uri config_hash
+        if config_hash[:user].nil?
+          mongo_uri = "mongodb://#{config_hash[:host]}#{config_hash[:port].nil? ? "" : ":#{config_hash[:port]}"}/#{config_hash[:database]}"
+        else
+          credentials = "#{config_hash[:username]}:#{config_hash[:password]}"
+          mongo_uri = "mongodb://#{config_hash[:host]}#{config_hash[:port].nil? ? "" : ":#{config_hash[:port]}"}@#{credentials}/#{config_hash[:database]}"
+        end
+        mongo_uri
+      end
+    end
+  end
+end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module DataAnonymization
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/spec/acceptance/mongodb_blacklist_spec.rb

@@ -0,0 +1,75 @@
+require "spec_helper"
+require 'mongo'
+
+describe "End 2 End MongoDB Blacklist Acceptance Test" do
+
+  before(:each) do
+    Mongo::Connection.from_uri("mongodb://localhost/test").drop_database('test')
+    users = [
+        {
+            "_id" => 1,
+            "user_id" => "sunitparekh",
+            "date_of_birth" => Time.new(2012, 7, 14, 13, 1, 0),
+            "email" => "parekh.sunit@gmail.com",
+            "password" => "TfqIK8Pd8GlbMDFZCX4l/5EtnOkfLCeynOL85tJQuxum&382knaflk@@",
+            "failed_attempts" => 0,
+            "first_name" => "Sunit",
+            "last_name" => "Parekh",
+            "password_reset_answer" => "manza",
+            "password_reset_question" => "My new car modal?",
+            "updated_at" => Time.new(2012, 8, 15, 13, 1, 0)
+        },
+        {
+            "_id" => 2,
+            "user_id" => "anandagrawal",
+            "date_of_birth" => Time.new(2011, 8, 11, 13, 1, 0),
+            "email" => "anandagrawal84@gmail.com",
+            "password" => "Tz548O0RWusldVAWkwqfzO3jK/X4l/5EtnOkfLCeynOL85tJQuxum",
+            "failed_attempts" => 0,
+            "first_name" => "Anand",
+            "last_name" => "Agrawal",
+            "password_reset_answer" => "android",
+            "password_reset_question" => "My phone?",
+            "updated_at" => Time.new(2012, 2, 11, 13, 1, 0)
+        }
+    ]
+    users_coll = Mongo::Connection.from_uri("mongodb://localhost/dest",{:safe => true})['test']['users']
+    users.each { |p| users_coll.save p }
+  end
+
+  it "should anonymize plans collection" do
+
+    database 'test' do
+      strategy DataAnon::Strategy::MongoDB::Blacklist
+      source_db :mongodb_uri => "mongodb://localhost/test", :database => 'test', :options => {:safe => true}
+
+      collection 'users' do
+        anonymize('date_of_birth').using FieldStrategy::TimeDelta.new(5,30)
+        anonymize('user_id').using FieldStrategy::StringTemplate.new('user-#{row_number}')
+        anonymize('email').using FieldStrategy::RandomMailinatorEmail.new
+        anonymize('password') { |field| "password" }
+        anonymize('first_name').using FieldStrategy::RandomFirstName.new
+        anonymize('last_name').using FieldStrategy::RandomLastName.new
+      end
+
+    end
+
+    users_coll = Mongo::Connection.from_uri("mongodb://localhost/test")['test']['users']
+    users_coll.count.should be 2
+    user = users_coll.find_one({'_id' => 1})
+
+    user['_id'].should == 1
+    user['user_id'].should == "user-1"
+    user['date_of_birth'].should_not == Time.new(2012, 7, 14, 13, 1, 0)
+    user['email'].should_not == "parekh.sunit@gmail.com"
+    user['password'].should == "password"
+    user['failed_attempts'].should == 0
+    user['first_name'].should_not be "Sunit"
+    user['last_name'].should_not be "Parekh"
+    user['password_reset_answer'].should == "manza"
+    user['password_reset_question'].should == "My new car modal?"
+    user['updated_at'].should == Time.new(2012, 8, 15, 13, 1, 0)
+
+
+  end
+end

data/spec/acceptance/mongodb_whitelist_spec.rb

@@ -0,0 +1,107 @@
+require "spec_helper"
+require 'mongo'
+
+describe "End 2 End MongoDB Whitelist Acceptance Test" do
+
+  before(:each) do
+    Mongo::Connection.from_uri("mongodb://localhost/test").drop_database('test')
+    Mongo::Connection.from_uri("mongodb://localhost/dest").drop_database('dest')
+    plans = [
+        {
+            "_id" => 1,
+            "name" => "Free",
+            "nick_names" => ["Name1","Name2"],
+            "features" => [
+                {
+                    "max_storage" => 21474836480,
+                    "type" => "AmazonS3",
+                    "users" => {"max" => 1, "additional" => false}
+                },
+                {
+                    "max_storage" => 21474836480,
+                    "type" => "DropBox",
+                    "users" => {"max" => 1, "additional" => false}
+                }
+            ],
+            "term" => "month",
+            "public_sharing" => false,
+            "photo_sharing" => true,
+            "created_at" => Time.new(2012, 6, 21, 13, 30, 0)
+        },
+        {
+            "_id" => 2,
+            "name" => "Team",
+            "plan_aliases" => ["Business", "Paid"],
+            "features" => [
+                {
+                    "max_storage" => 53687091200,
+                    "type" => "AmazonS3",
+                    "users" => {"max" => 5, "additional" => true}
+                },
+                {
+                    "max_storage" => 53687091200,
+                    "type" => "DropBox",
+                    "users" => {"max" => 5, "additional" => true}
+                }
+            ],
+            "term" => "month",
+            "public_sharing" => true,
+            "photo_sharing" => true,
+            "created_at" => Time.new(2012, 8, 11, 13, 1, 0)
+        }
+    ]
+    plans_coll = Mongo::Connection.from_uri("mongodb://localhost/dest")['test']['plans']
+    plans.each { |p| plans_coll.save p }
+  end
+
+  it "should anonymize plans collection" do
+
+    database 'test' do
+      strategy DataAnon::Strategy::MongoDB::Whitelist
+      source_db :mongodb_uri => "mongodb://localhost/test", :database => 'test'
+      destination_db :mongodb_uri => "mongodb://localhost/dest", :database => 'dest', :options => {:safe => true}
+
+      collection 'plans' do
+        whitelist '_id', 'name', 'term', 'created_at'
+        anonymize('plan_aliases').using FieldStrategy::SelectFromList.new(["Free", "Team", "Business", "Paid"])
+        anonymize 'public_sharing', 'photo_sharing'
+
+        collection 'features' do
+          anonymize('max_storage').using FieldStrategy::SelectFromList.new([10737418240, 21474836480, 53687091200])
+          whitelist 'type'
+
+          document 'users' do
+            anonymize 'max', 'additional'
+          end
+        end
+      end
+
+    end
+
+    plans_coll = Mongo::Connection.from_uri("mongodb://localhost/dest")['dest']['plans']
+    plans_coll.count.should be 2
+    plan = plans_coll.find_one({ '_id' => 1})
+
+    plan['_id'].should == 1
+    plan['name'].should == "Free"
+    plan['nick_names'][0].should_not == "Name1"
+    plan['nick_names'][1].should_not == "Name2"
+    plan['term'].should == "month"
+    plan['created_at'].should == Time.new(2012, 6, 21, 13, 30, 0)
+    plan['plan_aliases'].should be_nil
+    [true,false].should include(plan['public_sharing'])
+    [true,false].should include(plan['photo_sharing'])
+    plan['features'].length.should == 2
+    feature1 = plan['features'][0]
+    [10737418240, 21474836480, 53687091200].should include(feature1['max_storage'])
+    feature1['type'].should == "AmazonS3"
+    feature1['users']['max'].should be_kind_of(Fixnum)
+    [true,false].should include(feature1['users']['additional'])
+
+
+    plan = plans_coll.find_one({ '_id' => 2})
+    plan['plan_aliases'].length.should == 2
+    ["Free", "Team", "Business", "Paid"].should include(plan['plan_aliases'][0])
+    ["Free", "Team", "Business", "Paid"].should include(plan['plan_aliases'][1])
+  end
+end