db_obfuscation 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6336fb052f506c8e195330c28ba4c3f7ae082924
+  data.tar.gz: 9fd2a9f813622d45839461f45a4fb2d225d4bf6a
+SHA512:
+  metadata.gz: 1e93ff8b988dcbe559f17926fc71d361b25400b4937c75cc7948f9e2d1e50c89be7145d7ad9ff7b02a01eaa7eaf860dc31e7524cb453c5d17fe852e07e7a2dd0
+  data.tar.gz: e61875265eb209c2af9b0a8465c6f2c2ff3c394e2ee76ebf484b50cc99350134bdcc2b7cc8eef3b2d9bdd4ab1b73e7729d8bcbce698219fcba8a2ffb541e96f7

data/.gitignore

@@ -0,0 +1,6 @@
+logs
+log
+.vimlog
+db_schema.rb
+logfile
+test_logs

data/Gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+gem 'sequel'
+gem 'sequel_pg'
+gem 'thor'
+gem 'ffaker'
+gem 'pry-byebug'
+gem 'activesupport'
+
+group :test do
+  gem 'rspec'
+  gem 'aruba'
+end

data/Gemfile.lock

@@ -0,0 +1,83 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.0)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    aruba (0.6.2)
+      childprocess (>= 0.3.6)
+      cucumber (>= 1.1.1)
+      rspec-expectations (>= 2.7.0)
+    builder (3.2.2)
+    byebug (4.0.5)
+      columnize (= 0.9.0)
+    childprocess (0.5.6)
+      ffi (~> 1.0, >= 1.0.11)
+    coderay (1.1.0)
+    columnize (0.9.0)
+    cucumber (2.0.0)
+      builder (>= 2.1.2)
+      cucumber-core (~> 1.1.3)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.12)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.2)
+    cucumber-core (1.1.3)
+      gherkin (~> 2.12.0)
+    diff-lcs (1.2.5)
+    ffaker (2.0.0)
+    ffi (1.9.10)
+    gherkin (2.12.2)
+      multi_json (~> 1.3)
+    i18n (0.7.0)
+    json (1.8.2)
+    method_source (0.8.2)
+    minitest (5.5.1)
+    multi_json (1.11.1)
+    multi_test (0.1.2)
+    pg (0.18.2)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-byebug (3.1.0)
+      byebug (~> 4.0)
+      pry (~> 0.10)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.1)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    sequel (4.24.0)
+    sequel_pg (1.6.13)
+      pg (>= 0.8.0)
+      sequel (>= 3.39.0)
+    slop (3.6.0)
+    thor (0.19.1)
+    thread_safe (0.3.4)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  aruba
+  ffaker
+  pry-byebug
+  rspec
+  sequel
+  sequel_pg
+  thor

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Case Commons
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,121 @@
+# [db_obfuscation](https://github.com/CasecommonsDevops/db_obfuscation)
+
+
+`db_obfuscation` is a gem that helps to prepare a production size obfuscated database. This obfuscated database can be used for internal testing purposes like user acceptance testing, QA/Regression testing.
+
+`db_obfuscation` takes a production database and updates data in every row in each table with fake data. `db_obfuscation` ensures that associations between different tables are still maintained.
+
+**The gem supports only postgres databases at the moment.**
+
+## Installation
+
+`gem install db_obfuscation`
+
+
+## Usage
+
+```sh
+db_obfuscation obfuscate -c <path of obfuscation_configuration>
+                         -s <Number of rows to be obfuscated in each db transaction> #default 100
+                         -l <name_of_log_file>
+```
+
+`step_size` is a configuration that depends on every use case. It depends on the processing power of the computer, size of the table etc.
+
+In our experience, 100 row updates per database transaction has been the most optimum configuration for a database. However this number may need to be changed to optimize the performance for your database.
+
+
+## Configuration
+
+A sample configuration folder for the gem is included with the gem. The sample folder is at `spec/config`.
+
+A generic configuration folder consists of following files and folders,
+
+1. **Database Configuration file**
+
+  `<path_to_config_folder>/database.yml`
+
+  This file contains credentials to connect to the database. This file needs adapter name, host, encoding, username, password,  and name of the database.
+
+  Sample `database.yml` file:
+
+  ```yaml
+  adapter: postgres
+  host: localhost
+  encoding: unicode
+  username: database_user
+  database: obfuscation_test
+  password: database_password
+  ```
+
+2. **Table Strategies**
+
+  `<path_to_config_folder>/table_strategies`
+
+  This folder contains a yaml file for every table, for which a users desires to override default obfuscation configuration.
+
+  Each table file contains a mapping between columns and obfuscation strategy for that column. The filename is same as the table whose configuration is specified.
+
+  A sample table strategy file is like,
+
+  `<spec/config/table_strategies/table_2.yml>`
+
+  ```yaml
+  table_2:
+    field_1: :default_strategy
+    field_2: :whitelisted
+    date_field: :date_strategy
+    field_3: :first_name_strategy
+  ```
+
+  `db_obfuscation`, by default, obfuscates every **string** column in a table.
+
+  It uses a random word to obfuscate every string column. This default behaviour can be overridden on column and table basis by specifying different strategies respectively.
+
+    Different strategies supported are,
+
+    - `:whitelisted` to skip obfuscating a particular string column in a table
+    - `:date_strategy` to include a date column that needs to be obfuscated.
+
+      Date columns in a table are not obfuscated by default. Including `:date_strategy` adds a random number of days between 31 and 240 to the current value of date.
+    - Complete list of different strategies is [here](https://github.com/CaseCommonsDevOps/db_obfuscation/blob/master/lib/db_obfuscation/obfuscator.rb).
+
+3. **Truncation Tables**
+
+  `<path_to_config_folder>/truncation_patterns.yml`
+
+  This file contains string patterns for table names that need to truncated instead of being obfuscated.
+
+  Any table name that is the same as the pattern or begins with that pattern, followed by an underscore will be truncated during the obfuscation process.
+
+  A sample `truncation_patterns.yml` file is like,
+
+  ```yaml
+  - truncation_table_1
+  - audit
+  ```
+
+  Any table that begins with the word `audit_` will be selected for truncation.
+
+4. **Whitelisted Tables**
+
+  `<path_to_config_folder/whitelisted_tables.yml`
+
+  This file contains names of tables that don't need to be obfuscated and should not be touched.
+
+  A sample `whitelisted_tables.yml` looks like this,
+
+  ```yaml
+  - whitelisted_table_1
+  - whitelisted_table_2
+  ```
+
+## Requirements
+
+- Ruby 2.x
+
+## License
+
+Copyright © 2015 Case Commons & Rajat Agrawal.
+
+Licensed under the MIT license, available in the “LICENSE” file.

data/TODO

@@ -0,0 +1,15 @@
+1. Show the table number being obfuscated
+2. Don't obfuscate a table if the first update shows an exception
+3. Generalize triggers patterns
+4. Verify the configuration that has been specified by the user for obfuscation against the database. Like the column exists or not in the table strategy file
+5. Choosing verbosity of logs on the command line
+6. support both symbol and string use in user configuration files
+7. replace aruba cucumber with aruba rspec
+8. Don't fail in any of the user configuration is missing
+9. Write a task to generate a sample configuration
+10. Adding a dummy task
+
+
+Nice to have
+
+1. Ability to obfuscate a single table from the command line

data/bin/console

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+
+config_path = ARGV[0]
+if config_path.nil?
+  puts 'Please enter the obfuscation config folder path'
+  exit
+end
+
+require 'bundler'
+Bundler.setup(:default)
+
+lib_path = File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.unshift(lib_path) unless $LOAD_PATH.include?(lib_path)
+
+require 'db_obfuscation/environment'
+DbObfuscation.config_dir = Pathname.new(config_path)
+
+require 'db_obfuscation'
+require 'db_obfuscation/util/trigger'
+require 'pry'
+
+puts 'Console is starting'
+Pry.start

data/bin/db_obfuscation

@@ -0,0 +1,121 @@
+#!/usr/bin/env ruby
+
+require "thor"
+
+class DbObfuscationCli < Thor
+
+  desc 'dump', 'Take a dump of the obfuscated db'
+
+  option(:db_config,
+         aliases: :c,
+         required: true,
+         type: :string,
+         banner: 'db connection file for the db to be dumped')
+
+  option(:dump_name,
+         aliases: :n,
+         required: true,
+         type: :string,
+         banner: 'Name of the db dump',
+         description: <<-description.gsub(/\s{2,}/,' ')
+           A database dump helps to send the obfuscated dump
+           to another computer. Use this command after obfuscating
+           a database.
+           description
+         )
+  option(:log_file,
+         aliases: :l,
+         default: STDOUT,
+         type: :string,
+         banner: 'Name of log file for logging')
+  def dump
+    begin
+      require_relative '../lib/db_obfuscation/environment'
+      initialize_logger(options[:log_file])
+
+      require 'db_dump'
+
+      DbObfuscation.logging.info 'Start process'
+      DbObfuscation.logging.info 'Start dumping'
+      DbObfuscation::Cli::DbDump.dump(options[:db_config], options[:dump_name])
+      DbObfuscation.logging.info 'Finish dumping'
+
+    rescue Exception => e
+      DbObfuscation.logging.fatal 'Caught an exception'
+      DbObfuscation.logging.fatal e.message
+      DbObfuscation.logging.fatal e.backtrace
+    ensure
+      DbObfuscation.logging.info 'Process finished'
+    end
+  end
+
+  desc 'obfuscate', 'Obfuscates a database'
+  option(:config_path,
+         aliases: :c,
+         required: true,
+         type: :string,
+         banner: 'Folder path for obfuscation strategies')
+  option(:step_size,
+         aliases: :s,
+         default: 100,
+         type: :numeric,
+         banner: 'Number of rows to be obfuscated in every transaction')
+  option(:log_file,
+         aliases: :l,
+         default: STDOUT,
+         type: :string,
+         banner: 'Name of log file for logging')
+
+  def obfuscate
+    begin
+      require_relative '../lib/db_obfuscation/environment'
+      DbObfuscation.config_dir = Pathname.new(options[:config_path])
+      initialize_logger(options[:log_file])
+
+      DbObfuscation.logging.info 'Starting process'
+
+      require DbObfuscation::ROOT.join('lib/db_obfuscation')
+      require DbObfuscation::ROOT.join('lib/db_obfuscation/' + 'truncation')
+      require DbObfuscation::ROOT.join('lib/db_obfuscation/util/' + 'trigger')
+
+
+
+      DbObfuscation.logging.info 'Disabling triggers'
+      DbObfuscation::Util::Trigger.disable(:all)
+      DbObfuscation.logging.info 'Finished disabling triggers'
+
+      DbObfuscation.logging.info 'Truncating Tables'
+      DbObfuscation::Truncation.truncate
+      DbObfuscation.logging.info 'Finished truncating tables'
+
+
+      DbObfuscation.logging.info 'Starting DbObfuscation'
+      DbObfuscation.obfuscate(options[:step_size])
+      DbObfuscation.logging.info 'Finished DbObfuscation'
+
+    rescue Exception => e
+      DbObfuscation.logging.fatal 'Caught an exception'
+      DbObfuscation.logging.fatal e.message
+      DbObfuscation.logging.fatal e.backtrace
+    ensure
+      DbObfuscation.logging.info 'Finishing process'
+      DbObfuscation.logging.info 'Enabling triggers'
+      DbObfuscation::Util::Trigger.enable(:all)
+      DbObfuscation.logging.info 'Finished enabling triggers'
+
+      DbObfuscation.logging.info 'Process finished'
+    end
+  end
+
+  no_tasks do
+    def initialize_logger(filename)
+      if filename != STDOUT
+        f = File.new(filename, 'w')
+        f.close
+      end
+      DbObfuscation.logging = Logger.new(filename)
+    end
+  end
+end
+
+DbObfuscationCli.start

data/bin/obfuscation_test

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+require 'thor'
+require 'yaml'
+require_relative '../lib/db_obfuscation/environment'
+
+class ObfuscationTestCli < Thor
+
+  desc 'initialize_database', 'Initializes a test environment database'
+  option(:config,
+         aliases: :c,
+         type: :string,
+         default: DbObfuscation::ROOT.join('spec/config/database.yml'),
+         banner: 'Database yml containing the credentials to create/connect to the database')
+
+  option(:data_path,
+         aliases: :d,
+         type: :string,
+         default: DbObfuscation::ROOT.join('spec/test_db_setup'),
+         banner: 'Root folder path containing migrations and seed files')
+
+  def initialize_database
+    require 'pathname'
+    config = YAML.load_file(Pathname.new(options[:config]))
+    db_name = config['database']
+    owner = config['username']
+    migrations_path = Pathname.new(options[:data_path]).join('migrations')
+    seeds_path = Pathname.new(options[:data_path]).join('seeds')
+
+    puts "Dropping database #{db_name}"
+    system("dropdb -e #{db_name}")
+    puts 'Successfully dropped the database'
+
+    puts 'Creating database'
+    if owner
+      system("createdb -e -O #{owner} #{db_name}")
+    else
+      system("createdb -e #{db_name}")
+    end
+    puts 'Finished creating database'
+
+    puts 'Starting up migration'
+    require_relative '../cli/migrator'
+    DbObfuscation::Cli::Migrator.migrate(options[:config], migrations_path)
+    puts 'Finished migration'
+
+    puts 'Starting seeding'
+    require_relative '../cli/seeder'
+    DbObfuscation::Cli::Seeder.seed(options[:config], seeds_path)
+    puts 'Finished seeding'
+  end
+end
+
+ObfuscationTestCli.start