db_obfuscation 0.0.1

data/cli/db_dump.rb

@@ -0,0 +1,29 @@
+require 'yaml'
+
+module DbObfuscation
+  module Cli
+    module DbDump
+      def self.dump(config_file, dump_name)
+        config = db_config(config_file)
+        Kernel.system cmd(config, dump_name)
+      end
+
+      def self.db_config(file)
+        YAML.load_file(file)
+      end
+      private_class_method :db_config
+
+      def self.cmd(config, dump_name)
+        <<-CMD.gsub(/\s{2,}/,' ').strip
+        PGPASSWORD=#{config['password']}
+        pg_dump
+        -h #{config['host']}
+        -U #{config['username']} -w
+        -Fc -f #{dump_name}
+        #{config['database']}
+        CMD
+      end
+      private_class_method :cmd
+    end
+  end
+end

data/cli/migrator.rb

@@ -0,0 +1,26 @@
+require 'yaml'
+
+module DbObfuscation
+  module Cli
+    class Migrator
+      def self.migrate(config_file, path_to_migrations)
+        config = db_config(config_file)
+
+        db_name = "postgres://#{config['host']}/#{config['database']}"
+        system("sequel -m #{path_to_migrations} #{db_name}")
+      end
+
+      def self.down_migrate(config_file, path_to_migrations)
+        config = db_config(config_file)
+
+        db_name = "postgres://#{config['host']}/#{config['database']}"
+        system("sequel -m #{path_to_migrations} -M 0 #{db_name}")
+      end
+
+      def self.db_config(config_file)
+        YAML.load_file(config_file)
+      end
+      private_class_method :db_config
+    end
+  end
+end

data/cli/seeder.rb

@@ -0,0 +1,52 @@
+require 'yaml'
+require 'pathname'
+require 'sequel'
+
+module DbObfuscation
+  module Cli
+    class Seeder
+
+      def initialize(config_file, path_to_seeds)
+        @db_connection = db_connection(config_file)
+        files = seed_files(path_to_seeds)
+        @seed_data = seed_data(files)
+      end
+
+      def self.seed(*args)
+        new(*args).seed
+      end
+
+      def seed
+        @seed_data.each do |table, data|
+          data.each do |row|
+            @db_connection[table].insert(row)
+          end
+        end
+      end
+
+      private
+
+      def db_connection(config_file)
+        config = YAML.load_file(config_file)
+        Sequel.connect(config)
+      end
+
+      def seed_files(path)
+        seeds_path = Pathname.new(path).join('*')
+        Dir[seeds_path]
+      end
+
+      def table_name(file)
+        File.basename(file, '.yml').to_sym
+      end
+
+      def seed_data(files)
+        files.each_with_object({}) do |file, data|
+          table = table_name(file)
+          values = YAML.load_file(file).map { |_,v| v }
+          data[table] = values
+        end
+      end
+    end
+  end
+end

data/db_obfuscation.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'db_obfuscation/version'
+
+Gem::Specification.new do |spec|
+  spec.name        = 'db_obfuscation'
+  spec.version     = DbObfuscation::VERSION
+  spec.authors     = ['Case Commons, LLC', 'Rajat Agrawal']
+  spec.email       = ['casebook-dev@googlegroups.com', 'agrawal.rajat.89@gmail.com']
+  spec.homepage    = 'https://github.com/CaseCommonsDevOps/db_obfuscation'
+  spec.date        = '2015-07-21'
+  spec.summary     = "A gem to obfuscate a production database with fake values for testing with a production size database"
+  spec.description = <<-description.gsub(/\s{2,}/, ' ')
+    db_obfuscation is a gem that helps to prepare a production size obfuscated database. This obfuscated database can be used for internal testing purposes like user acceptance testing, QA/Regression testing.
+
+    db_obfuscation takes a production database and updates data in every row in each table with fake data. db_obfuscation ensures that associations between different tables are still maintained.
+    description
+  spec.license     = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/db_obfuscation}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+end

data/features/bin/dump.feature

@@ -0,0 +1,21 @@
+Feature: Dump
+  This is a command to compress an obfuscated database
+  A compressed db is easier to transport
+  It takes a config path for the database connection and
+  Name of the compressed database dump
+
+  Scenario: Successful database dump
+
+    When I run `../../bin/db_obfuscation dump -c ../../spec/config/database.yml -n test_compressed_db`
+    Then the output should contain "Start process"
+    Then the output should contain "Start dumping"
+    Then the output should contain "Finish dumping"
+    Then the output should contain "Process finished"
+    Then the output should not contain "Caught an exception"
+
+  Scenario: Unsuccessful database dump
+
+    When I run `../../bin/db_obfuscation dump -c incorrect_database.yml -n test_compressed_db`
+    Then the output should contain "Start process"
+    Then the output should contain "Process finished"
+    Then the output should contain "Caught an exception"

data/features/bin/obfuscation.feature

@@ -0,0 +1,12 @@
+Feature: Db_Obfuscation
+  This is a command line thor task to run obfuscation for a user
+  It takes a folder path for obfuscation configuration
+  And obfuscates the database
+
+  Scenario: Successfully Obfuscating a Database
+    When I run `../../bin/db_obfuscation obfuscate -c ../../spec/config/ -s 200`
+    Then the exit status should be 0
+
+  Scenario: Unsuccessfully obfuscating a Database
+    When I run `../../bin/db_obfuscation obfuscate -c incorrect_config_folder -s 200`
+    Then the exit status should not be 0

data/features/bin/test_database_tasks.feature

@@ -0,0 +1,16 @@
+Feature: Test Database tasks
+  Scenario: Preparing a test database
+  Given I run `createdb obfuscation_test`
+
+  When I run `../../bin/obfuscation_test initialize_database -c ../../spec/config/database.yml -d ../../spec/test_db_setup`
+  Then the output should contain "Dropping database obfuscation_test"
+  Then the output should contain "DROP DATABASE obfuscation_test;"
+  Then the output should contain "Successfully dropped the database"
+
+  Then the output should contain "Creating database"
+  Then the output should contain "CREATE DATABASE obfuscation_test OWNER"
+  Then the output should contain "Finished creating database"
+  Then the output should contain "Starting up migration"
+  Then the output should contain "Finished migration"
+  Then the output should contain "Starting seeding"
+  Then the output should contain "Finished seeding"

data/features/support.rb

@@ -0,0 +1 @@
+require 'aruba/cucumber'

data/lib/db_obfuscation.rb

@@ -0,0 +1,50 @@
+require 'db_obfuscation/environment'
+require 'db_obfuscation/database'
+require 'db_obfuscation/filtering'
+require 'db_obfuscation/query_builder'
+require 'db_obfuscation/batch_formulator'
+
+module DbObfuscation
+  extend self
+
+  def obfuscate(step)
+    config = Filtering.obfuscation_config([:string])
+    config.each do |table, cfg|
+      DbObfuscation.logging.info "Obfuscating #{table}"
+      update(table, cfg, step)
+    end
+  end
+
+  private
+
+  def update(table, config, step)
+    ids(table).each_slice(step) do |ids|
+      begin
+        batch = BatchFormulator.batch_for(config, ids)
+        date_columns = date_columns(config)
+        multi_update(table, batch, date_columns) unless batch.empty?
+      rescue => e
+        DbObfuscation.logging.error 'Encountered Exception'
+        DbObfuscation.logging.error "#{table} encountered #{e.message}"
+        DbObfuscation.logging.error e.backtrace
+      end
+    end
+  end
+
+  def date_columns(config)
+    config.select do |k,v|
+      v == :date_strategy
+    end.keys
+  end
+
+  def ids(table)
+    DB[table].map(:id)
+  end
+
+  def multi_update(table, batch, date_columns)
+    sql_query = QueryBuilder.multi_update_sql(table,
+                                              batch,
+                                              date_columns)
+    DB.run sql_query
+  end
+end

data/lib/db_obfuscation/batch_formulator.rb

@@ -0,0 +1,26 @@
+require 'db_obfuscation/database'
+require 'db_obfuscation/obfuscator'
+
+module DbObfuscation
+  class BatchFormulator
+
+    def self.batch_for(*args)
+      self.new.batch_for(*args)
+    end
+
+    def batch_for(config, ids)
+      ids.each_with_object([]) do |id, batch|
+        batch<<obfuscate_row(config, id)
+      end
+    end
+
+    private
+
+    def obfuscate_row(config, id)
+      config.each_with_object({}) do |(column, strategy), obfuscated_hash|
+        value = Obfuscator.obfuscate(strategy)
+        obfuscated_hash[column] = DB.literal value
+      end.merge(id: id)
+    end
+  end
+end

data/lib/db_obfuscation/config.rb

@@ -0,0 +1,43 @@
+require 'yaml'
+require 'pathname'
+
+module DbObfuscation
+  class Config
+    class << self
+      def config_path
+        @config || DbObfuscation.config_dir
+      end
+
+      def config_path=(config)
+        @config = Pathname.new(config)
+      end
+
+      def db_config
+        YAML.load_file(config_path.join('database.yml'))
+      end
+
+
+      def whitelisted_tables
+        YAML.load_file(config_path.join('whitelisted_tables.yml'))
+      end
+
+      def table_strategies
+        @@table_strategies ||= load_table_strategies
+      end
+
+      def truncation_patterns
+        @truncation_tables ||= YAML.load_file(config_path.join('truncation_patterns.yml')).map(&:to_sym)
+      end
+
+      private
+      def load_table_strategies
+        strategies = {}
+        file_path = "#{config_path}/table_strategies/*.yml"
+        Dir[file_path].each do |file|
+          strategies.merge!(YAML.load_file(file)) if File.file?(file)
+        end
+        strategies
+      end
+    end
+  end
+end

data/lib/db_obfuscation/database.rb

@@ -0,0 +1,8 @@
+require 'sequel'
+require 'db_obfuscation/config'
+
+module DbObfuscation
+  DB = Sequel.connect(Config.db_config)
+  Sequel.default_timezone = :utc
+  Sequel.extension :migration
+end

data/lib/db_obfuscation/environment.rb

@@ -0,0 +1,14 @@
+require 'pathname'
+require 'logger'
+
+module DbObfuscation
+  ROOT = Pathname.new(File.expand_path('../../../', __FILE__)).freeze
+  class << self
+    attr_accessor :config_dir, :logging
+  end
+end
+
+lib_path = DbObfuscation::ROOT.join('lib')
+cli_path = DbObfuscation::ROOT.join('cli')
+$LOAD_PATH.unshift(lib_path) unless $LOAD_PATH.include?(lib_path)
+$LOAD_PATH.unshift(cli_path) unless $LOAD_PATH.include?(cli_path)

data/lib/db_obfuscation/filtering.rb

@@ -0,0 +1,56 @@
+require 'db_obfuscation/database'
+require 'db_obfuscation/filtering/column'
+require 'db_obfuscation/truncation'
+require 'db_obfuscation/obfuscation_strategy'
+require 'active_support/core_ext/object/blank'
+
+module DbObfuscation
+  module Filtering
+    extend self
+
+    def obfuscation_config(types)
+      filter_tables.each_with_object({}) do |table, config|
+        table_config = config_per_table(table, types)
+        config[table] = table_config if table_config.present?
+      end
+    end
+
+    private
+
+    def config_per_table(table, types)
+      table_config = Column.columns_type(table).each_with_object({}) do |(column_name, column_type), config|
+        filtered_column = Column.filter(column_name, column_type, types)
+        config[column_name] = default_obfuscation_strategy(filtered_column) if filtered_column
+      end.merge(user_config(table))
+
+      reject_whitelisted_columns(table_config)
+    end
+
+    def default_obfuscation_strategy(column)
+      ObfuscationStrategy.strategy(column)
+    end
+
+    def filter_tables
+      DbObfuscation::DB.tables - exclude_tables
+    end
+
+    def user_config(table)
+      table_strategy = DbObfuscation::Config.table_strategies[table.to_s] || {}
+      table_strategy.each_with_object({}) { |(k,v), config| config[k.to_sym] = v }
+    end
+
+    def reject_whitelisted_columns(config)
+      config.reject { |k,v| v == :whitelisted }
+    end
+
+    def exclude_tables
+      DbObfuscation::Truncation.tables +
+      whitelisted_tables +
+      [:schema_info]
+    end
+
+    def whitelisted_tables
+      DbObfuscation::Config.whitelisted_tables.map(&:to_sym)
+    end
+  end
+end

data/lib/db_obfuscation/filtering/column.rb

@@ -0,0 +1,40 @@
+require 'db_obfuscation/database'
+
+module DbObfuscation
+  module Filtering
+    module Column
+      class << self
+        def columns_type(table_name)
+          column_types = {}
+          columns(table_name).each do |column_name, column_details|
+            column_types[column_name] = column_details[:type]
+          end
+          column_types
+        end
+
+        def columns(table_name)
+          DbObfuscation::DB.schema(table_name)
+        end
+
+        def type?(expected_column_types, column_type)
+          expected_column_types.include? column_type
+        end
+
+        def polymorphic?(column_name)
+          column_name.match(/type$/) ? true : false
+        end
+
+        def ending_in_id?(column_name)
+          column_name.match(/id$/) ? true : false
+        end
+
+        def filter(column_name, column_type, expected_types)
+          return nil if polymorphic?(column_name)
+          return nil if ending_in_id?(column_name)
+          return nil unless type?(expected_types, column_type)
+          return column_name
+        end
+      end
+    end
+  end
+end

data/lib/db_obfuscation/filtering/truncation.rb

@@ -0,0 +1,18 @@
+module DbObfuscation
+  module Filtering
+    class Truncation
+
+      def self.matches_patterns(*args)
+        new.send(:matches_patterns, *args)
+      end
+
+      private
+
+      def matches_patterns(tables, patterns)
+        patterns.map do |pattern|
+          tables.grep(/^#{pattern}(_.*)*$/)
+        end.flatten.uniq
+      end
+    end
+  end
+end