active_sanitization 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 621371200d0de25b03a94a5cc486bbfd8438950f
+  data.tar.gz: 96dd6d55756ac5d0cfa26c0eda7aa24099f31e1b
+SHA512:
+  metadata.gz: 83d4f4053ac906dd77c8c2805f1e03ad524a54205bd8484086b2ade857ad48b8a7f90b54769a6f0a304c4df49719f99a2ef5c4d7861e64f31f4a61d8c05fbac8
+  data.tar.gz: f0b10f1d68af8e9e2028007dac8f1a4ad26b2692a287b4f2c3d465998e28da66760bb82e887eac2e6337d45b4c4be6b570f338f817693dcdeacecaf635798b09

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,7 @@
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.2.1
+before_script:
+  - mkdir tmp

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# 0.1.0
+Initial release
+  - Sets up gem
+  - Adds dependencies
+  - Exposes rake tasks

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in active_sanitization.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,157 @@
+PATH
+  remote: .
+  specs:
+    active_sanitization (0.1.0)
+      aws-sdk (~> 2.0.33)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionmailer (4.2.1)
+      actionpack (= 4.2.1)
+      actionview (= 4.2.1)
+      activejob (= 4.2.1)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+    actionpack (4.2.1)
+      actionview (= 4.2.1)
+      activesupport (= 4.2.1)
+      rack (~> 1.6)
+      rack-test (~> 0.6.2)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.1)
+    actionview (4.2.1)
+      activesupport (= 4.2.1)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.1)
+    activejob (4.2.1)
+      activesupport (= 4.2.1)
+      globalid (>= 0.3.0)
+    activemodel (4.2.1)
+      activesupport (= 4.2.1)
+      builder (~> 3.1)
+    activerecord (4.2.1)
+      activemodel (= 4.2.1)
+      activesupport (= 4.2.1)
+      arel (~> 6.0)
+    activesupport (4.2.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    arel (6.0.0)
+    aws-sdk (2.0.33)
+      aws-sdk-resources (= 2.0.33)
+    aws-sdk-core (2.0.33)
+      builder (~> 3.0)
+      jmespath (~> 1.0)
+      multi_json (~> 1.0)
+      multi_xml (~> 0.5)
+    aws-sdk-resources (2.0.33)
+      aws-sdk-core (= 2.0.33)
+    builder (3.2.2)
+    byebug (4.0.4)
+      columnize (= 0.9.0)
+    coderay (1.1.0)
+    columnize (0.9.0)
+    diff-lcs (1.2.5)
+    erubis (2.7.0)
+    globalid (0.3.3)
+      activesupport (>= 4.1.0)
+    hike (1.2.3)
+    i18n (0.7.0)
+    jmespath (1.0.2)
+      multi_json (~> 1.0)
+    json (1.8.2)
+    loofah (2.0.1)
+      nokogiri (>= 1.5.9)
+    mail (2.6.3)
+      mime-types (>= 1.16, < 3)
+    method_source (0.8.2)
+    mime-types (2.4.3)
+    mini_portile (0.6.2)
+    minitest (5.5.1)
+    multi_json (1.11.0)
+    multi_xml (0.5.5)
+    mysql2 (0.3.18)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rack (1.6.0)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rails (4.2.1)
+      actionmailer (= 4.2.1)
+      actionpack (= 4.2.1)
+      actionview (= 4.2.1)
+      activejob (= 4.2.1)
+      activemodel (= 4.2.1)
+      activerecord (= 4.2.1)
+      activesupport (= 4.2.1)
+      bundler (>= 1.3.0, < 2.0)
+      railties (= 4.2.1)
+      sprockets-rails
+    rails-deprecated_sanitizer (1.0.3)
+      activesupport (>= 4.2.0.alpha)
+    rails-dom-testing (1.0.6)
+      activesupport (>= 4.2.0.beta, < 5.0)
+      nokogiri (~> 1.6.0)
+      rails-deprecated_sanitizer (>= 1.0.1)
+    rails-html-sanitizer (1.0.2)
+      loofah (~> 2.0)
+    railties (4.2.1)
+      actionpack (= 4.2.1)
+      activesupport (= 4.2.1)
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    rake (10.4.2)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.1)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
+    slop (3.6.0)
+    sprockets (2.12.3)
+      hike (~> 1.2)
+      multi_json (~> 1.0)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    sprockets-rails (2.2.4)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      sprockets (>= 2.8, < 4.0)
+    sqlite3 (1.3.10)
+    thor (0.19.1)
+    thread_safe (0.3.5)
+    tilt (1.4.1)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  active_sanitization!
+  activerecord (~> 4.2.1)
+  bundler (~> 1.8.3)
+  byebug (~> 4.0.4)
+  mysql2 (~> 0.3.18)
+  pry (~> 0.10.1)
+  rails (~> 4.2.1)
+  rake (~> 10.0)
+  rspec (~> 3.2.0)
+  sqlite3 (~> 1.3.10)

data/LICENSE.txt

@@ -0,0 +1,19 @@
+The MIT License (MIT)
+ 
+Copyright (c) 2014 RightScale, Inc, All Rights Reserved Worldwide.
+ 
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,194 @@
+# Active Sanitization
+
+Active Sanitization provides an easy way to sanitize your mysql database.  By using the configuration options you are able customize what gets sanitized, truncated and ignored.  You can also provide S3 creds, that will allow the sanitized snapshot to be uploaded.  This makes it easy for everyone to get access to this snapshot without having to go through production access gatekeeper.
+
+## Features
+### How it works
+Active Sanitization works by copying your current database into a temporary one.  Once this is done it allows a series of santizations to be performed.
+Before any of this happens it checks that the database doesn't contain any extra tables or columns that it doesn't know about.  This means that there is no way an extra column can be added without the correct sanitization being added. (Please see the Sanitization tests section for more details).
+
+Once all the pre_sanitization checks have been performed and passed, then the sanitization process can begin.  Active Sanitization will copy the content of all tables that require sanitizationand all tables that should be truncated (without data) to a temporary database.  The sanitization process is relatively simple.  It will loop through all tables, and if that table has any columns that require sanitization then it will do that.  It does that by swapping all distinct values for each column that requires sanitization with a random value from the allowed sanitized substitutes.  Once it has done this it will call through to a Custom Santization class that you provide. (Please see usage for how this works).
+
+After all tables have been sanitized then the the temporary table will be exported to a tmp directory, before being dropped.
+
+### S3 Upload
+
+There is an option to upload the sanitized snapshot to S3, and fetch it again.  To do this simply provide the following values in the ActiveSantization config
+```
+s3_bucket
+s3_bucket_region
+aws_access_key_id
+aws_secret_access_key
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'active_sanitization'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install active_sanitization
+
+## Usage
+
+To use Active Sanitization add the following to your Rakefile
+```
+require 'active_sanitization'
+```
+
+Next you will need to configure the gem.  This should be done in an initializer:
+```ruby
+# After going through a tables standard sanitization ActiveSanitization can call out to a custom bit of code.
+# This allows you to provide varies custom sanitizations that require a bit more in depth knowledge of the data involved.
+# To do this you should provide a class, that contains the custom sanitization code.  ActiveSantization will call any function named
+# `santize_TABLE_NAME`, and will pass is an ActiveRecord::Base.connection to the temporary database (that contains data that is being sanitized)
+# The example bellow removes all rows from the `people` table who have an age less than 22.
+class CustomSanitization
+  def self.sanitize_person(temp_db_connection)
+    temp_db_connection.execute("DELETE FROM people WHERE age < 22")
+  end
+end
+
+ActiveSanitization.configure do |config|
+  # Tables that need to be sanitized
+  # This is a hash, where the key is each table name, and the value being an array of
+  #    of all the columns in that table
+  config.tables_to_sanitize = {
+    "people" => ["address", "age", "gender", "id", "name"],
+    "cars"   => ["id", "make", "model", "number_of_doors"],
+  }
+
+  # Tables that need to be truncated
+  # This is a hash, where the key is each table name, and the value being an array of
+  #    of all the columns in that table
+  # After the sanitization process these tables will exist in the database dump, but they will be empty
+  config.tables_to_truncate = {
+    "hotels" => ["address", "id", "name", "number_of_rooms"]
+  }
+
+  # This is an array of all the other tables in the database that will be ignored.
+  # These will not be exported or have any sanitization applied to them
+  config.tables_to_ignore = []
+
+  # This is a hash of standard sanitizations that are applied to all columns with the same name as the key
+  #   This is a hash where the key is the name of the column that needs to be sanitized, and the values what
+  #   the values in the column are going to be replaced with.
+  # For example every column called `name` will have the values replaced randomly as on the following values `['Tony', 'Adam', 'Claire', 'Sarah']`
+  config.sanitization_columns = {
+    'name' => ['Tony', 'Adam', 'Claire', 'Sarah'],
+    'make' => ['BMW', 'Toyota']
+  }
+
+  # This is the active_record_connection to the mysql database.  The connection to the correct database should already be established
+  config.active_record_connection = ActiveRecord::Base.connection
+
+  # The current environment that your application is running in
+  config.env = "test"
+
+  # This is the database config, that ActiveRecord used to connect to the database.
+  #   This is needed so we can establish a second connection to a temporary database (where we perform the sanitization)
+  config.db_config = {
+   'host'     => "localhost",
+   'username' => "root",
+   'password' => nil,
+   'database' => "active_sanitization",
+   'adapter' => "mysql2",
+  }
+
+  # The name of your app
+  config.app_name = 'super_secret_app'
+
+  # The logger that the gem should use.
+  #  This will default to STOUT if non is provided
+  config.logger = Rails.logger
+
+  # The path to the root of your project
+  #  This is required so the database dump can be put in a tmp folder
+  config.root = File.dirname(File.dirname(__FILE__))
+
+  # This is a class that you provide to do custom sanitization
+  config.custom_sanitization = CustomSanitization
+
+  # Upload to S3.
+  # There is an option to upload the sanitized database dump
+  config.s3_bucket = S3_BUCKET
+  config.s3_bucket_region = 'us-east-1'
+  config.aws_access_key_id = AWS_ACCESS_KEY_ID
+  config.aws_secret_access_key = AWS_SECRET_ACCESS_KEY
+end
+```
+### Sanitization tests
+
+Active Sanitization provides an easy way to add a test that will fail if the config is not updated to include all tables and columns.
+This can be done like:
+```
+require 'spec_helper'
+
+describe ActiveSanitization do
+  context ".pre_sanitization_checks" do
+    context "no new tables or columns have been added" do
+      # This will fail if a new column or table has been added but the hashes haven't been updated
+      it "doesn't stop" do
+        expect(ActiveSanitization.pre_sanitization_checks).to eq({
+          :pass => true
+        })
+      end
+    end
+  end
+end
+```
+
+It is also good to add tests for any custom sanitizations that you add.  Once you have loaded your data into the database, you can call:
+```
+@temp_db, @temp_db_connection, @temp_db_config = ActiveSanitization.duplicate_database
+ActiveSanitization.sanitize_tables(@temp_db_connection)
+```
+This will duplicate the database and call the sanitization code.  After this has run you should assert that your custom sanitization has performed as expected.
+
+## Actually using the gem
+
+This can be done using two rake tasks that ActiveSanitization provides.
+```
+rake active_sanitization:import_data_from_s3[env,timestamp]
+    Import sanitized data from S3 into MySQL.  Optional arguments are `env` and `timestamp`.  These will default to 'production' and the latest snapshot if they are not provided
+
+rake active_sanitization:sanitize_and_export_data
+    Sanitises MySQL database. If S3 creds are provided then the sanitized snapshot will be uploaded to S3
+```
+
+## Running the specs
+
+This is the default rake task so you can run the specs in any of the following ways:
+
+```bash
+bundle exec rake
+bundle exec rake spec
+```
+
+## Getting a console
+
+The project is currently using pry. In order to get a console in the context of the project just run the pry.rb file in ruby.
+
+```bash
+bundle exec rake console
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/active_sanitization/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## Maintained by
+
+- [Stephen Haley](https://github.com/shaley91)

data/Rakefile

@@ -0,0 +1,15 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+require 'active_sanitization'
+
+RSpec::Core::RakeTask.new('spec')
+Dir[File.dirname(__FILE__) + '/lib/tasks/**/*.rake'].each { |file| import file }
+
+task :default => :spec
+
+desc "console"
+task :console do
+  require 'pry'
+  binding.pry
+end

data/active_sanitization.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'active_sanitization/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "active_sanitization"
+  spec.version       = ActiveSanitization::VERSION
+  spec.authors       = ["Stephen Haley", "Callum Dryden"]
+  spec.email         = ["stephen.haley@rightscale.com", "callum.dryden@rightscale.com"]
+
+  spec.summary       = %q{Quick in-place santization for your MySql DB}
+  spec.description   = %q{Active Santization provides any easy way to consistently sanitize data from a MySql DB.  With the config you can specify how to sanitize each tables and column.}
+  spec.homepage      = "https://github.com/rightscale/active_sanitization"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.8.3"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.2.0"
+  spec.add_development_dependency "rails", "~> 4.2.1"
+  spec.add_development_dependency "pry", "~> 0.10.1"
+  spec.add_development_dependency "sqlite3", "~> 1.3.10"
+  spec.add_development_dependency "activerecord", "~> 4.2.1"
+  spec.add_development_dependency "mysql2", "~> 0.3.18"
+  spec.add_development_dependency "byebug", "~> 4.0.4"
+
+  spec.add_runtime_dependency "aws-sdk", "~> 2.0.33"
+end

data/lib/active_sanitization/version.rb

@@ -0,0 +1,3 @@
+module ActiveSanitization
+  VERSION = "0.1.0"
+end

data/lib/active_sanitization.rb

@@ -0,0 +1,293 @@
+require "active_sanitization/version"
+require_relative "tasks/rake_tasks"
+require "active_record"
+require "active_support"
+require 'aws-sdk'
+
+module ActiveSanitization
+  class << self
+    attr_accessor :configuration
+  end
+
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield(configuration)
+  end
+
+  class Configuration
+    attr_accessor :tables_to_sanitize, :tables_to_truncate, :tables_to_ignore, :sanitization_columns, :s3_bucket, :app_name, :aws_access_key_id, :aws_secret_access_key, :env, :active_record_connection, :db_config, :custom_sanitization, :logger, :root, :s3_bucket_region
+
+    def initialize
+      @tables_to_sanitize = {}
+      @tables_to_truncate = {}
+      @tables_to_ignore = {}
+      @sanitization_columns = {}
+      @s3_bucket = 'active_sanitization'
+      @env = ENV['RACK_ENV'] || ENV['RAILS_ENV']
+      @active_record_connection = ActiveRecord::Base.connection
+      @root = File.dirname(File.dirname(__FILE__))
+      @logger = Logger.new(STDOUT)
+    end
+  end
+
+  # Need to create a second ActiveRecord::Base.connection so we can
+  # connect to the primary and copy DB.
+  class TempDatabaseConnection < ActiveRecord::Base
+    def self.abstract_class?
+      true # So it gets its own connection
+    end
+  end
+
+  # Returns a hash that represents the difference between two hashes.
+  #
+  #   hash_diff({1 => 2}, {1 => 2})         # => {}
+  #   hash_diff({1 => 2}, {1 => 3})         # => {1 => 2}
+  #   hash_diff({}, {1 => 2})               # => {1 => 2}
+  #   hash_diff({1 => 2, 3 => 4}, {1 => 2}) # => {3 => 4}
+  def self.hash_diff(hash1, hash2)
+    difference1 = hash1.dup
+    difference2 = hash2.dup
+
+    difference1.delete_if do |key, value|
+      hash2[key] == value
+    end
+
+    difference2.delete_if do |key, value|
+      hash1.has_key?(key)
+    end
+
+    difference1.merge(difference2)
+  end
+
+  def self.log(output)
+    self.configuration.logger.info(output) unless self.configuration.env == 'test'
+  end
+
+  def self.pre_sanitization_checks
+    db_tables = {}
+    self.configuration.active_record_connection.tables.each do |table_name|
+      next if self.configuration.tables_to_ignore.include?(table_name)
+      db_tables[table_name] = []
+      self.configuration.active_record_connection.columns(table_name).each { |c| db_tables[table_name] << c.name }
+      db_tables[table_name].sort!
+    end
+
+    # diff will only work correctly if the columns are sorted the same
+    tables_with_sorted_columns = {}
+    self.configuration.tables_to_sanitize.merge(self.configuration.tables_to_truncate).each { |k, v| tables_with_sorted_columns[k] = v.sort }
+    table_difference = hash_diff(db_tables, tables_with_sorted_columns)
+    checks = {}
+    if table_difference != {}
+      column_difference = {}
+      table_difference.collect do |table_name, table_columns|
+        column_difference[table_name] = table_columns - self.configuration.tables_to_sanitize.merge(self.configuration.tables_to_truncate)[table_name].to_a
+      end
+      checks[:pass] = false
+      checks[:error] = "The following tables or columns have been found in the #{self.configuration.env} DB but are not known to this script (#{column_difference}).\n Please update the active_sanitization config!"
+    else
+      checks[:pass] = true
+    end
+    checks
+  end
+
+  def self.duplicate_database
+    temp_db = "#{self.configuration.db_config['database']}_copy"
+
+    self.log("Deleting temp DB if exists")
+    self.configuration.active_record_connection.execute("DROP DATABASE IF EXISTS #{temp_db};")
+    self.log("Creating temp DB")
+    self.configuration.active_record_connection.execute("CREATE DATABASE #{temp_db}")
+    self.log("Copying #{self.configuration.env} DB to temp DB")
+    self.log("mysqldump -h #{self.configuration.db_config['host']} -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} #{self.configuration.db_config['database']} #{self.configuration.tables_to_sanitize.keys.join(' ')} | mysql -h #{self.configuration.db_config['host']}  -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} -D #{temp_db}")
+    system("mysqldump -h #{self.configuration.db_config['host']} -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} #{self.configuration.db_config['database']} #{self.configuration.tables_to_sanitize.keys.join(' ')} | mysql -h #{self.configuration.db_config['host']}  -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} -D #{temp_db}")
+    if $?.exitstatus == 0
+      self.log("Temp DB created and populated")
+    else
+      raise "Failed to load DB #{self.configuration.db_config} into temp DB #{temp_db}."
+    end
+
+    self.log("mysqldump -h #{self.configuration.db_config['host']} -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} --no-data #{self.configuration.db_config['database']} #{self.configuration.tables_to_truncate.keys.join(' ')} | mysql -h #{self.configuration.db_config['host']}  -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} -D #{temp_db}")
+    system("mysqldump -h #{self.configuration.db_config['host']} -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} --no-data #{self.configuration.db_config['database']} #{self.configuration.tables_to_truncate.keys.join(' ')} | mysql -h #{self.configuration.db_config['host']}  -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} -D #{temp_db}")
+    if $?.exitstatus == 0
+      self.log("Temp DB created and populated")
+    else
+      raise "Failed to load DB #{self.configuration.db_config} into temp DB #{temp_db}."
+    end
+
+    temp_db_config = self.configuration.db_config.dup
+    temp_db_config['database'] = temp_db
+    TempDatabaseConnection.establish_connection(temp_db_config)
+    temp_db_connection = TempDatabaseConnection.connection
+
+    [temp_db, temp_db_connection, temp_db_config]
+  end
+
+  def self.sanitize_table(table, temp_db_connection)
+    table_columns = temp_db_connection.select_values("DESCRIBE #{table};")
+    self.configuration.sanitization_columns.keys.each do |column|
+      if table_columns.include?(column)
+        distinct_values = temp_db_connection.execute("SELECT DISTINCT(#{column}) FROM #{table};").collect { |data| data.first }
+        distinct_values.each do |value|
+          temp_db_connection.execute("UPDATE #{table} SET #{column}='#{self.configuration.sanitization_columns[column].sample}' WHERE #{column}=#{ActiveRecord::Base.sanitize(value)};")
+        end
+      end
+    end
+
+    # Run any custom sanitization for the table
+    self.configuration.custom_sanitization.send("sanitize_#{table}", temp_db_connection) if self.configuration.custom_sanitization.respond_to?("sanitize_#{table}")
+  end
+
+  def self.create_files
+    dump_file = "#{File.join(self.configuration.root, "tmp")}/data.dump"
+    compressed_dump_file = "#{dump_file}.gz"
+    File.new(dump_file,  "w+")
+    File.new(compressed_dump_file,  "w+")
+    [dump_file, compressed_dump_file]
+  end
+
+  def self.sanitize_tables(temp_db_connection)
+    self.log("Processing TABLES_TO_TRUNCATE...")
+    self.configuration.tables_to_truncate.keys.each do |table|
+       self.log("Truncating #{table}")
+       temp_db_connection.execute("TRUNCATE #{table};")
+    end
+
+    self.log("Processing TABLES_TO_SANITIZE...")
+    self.configuration.tables_to_sanitize.keys.each do |table|
+      self.log("Sanitizing #{table}")
+      self.sanitize_table(table, temp_db_connection)
+    end
+  end
+
+  def self.clean_up_temp_db(temp_db)
+    self.log("Dropping #{temp_db}")
+    self.configuration.active_record_connection.execute("DROP DATABASE #{temp_db};")
+  end
+
+  def self.gzip(dump_file)
+    self.log("Gzipping #{dump_file}")
+    system("gzip '#{dump_file}'")
+  end
+
+  def self.get_s3_bucket
+    creds = Aws::Credentials.new(self.configuration.aws_access_key_id, self.configuration.aws_secret_access_key)
+    client = Aws::S3::Client.new(credentials: creds, region: self.configuration.s3_bucket_region)
+    resource = Aws::S3::Resource.new(client: client)
+    resource.bucket(self.configuration.s3_bucket)
+  end
+
+  def self.upload(compressed_dump_file)
+    timestamp = DateTime.now.strftime('%Y%m%d%H%M%S')
+    name = "#{self.configuration.app_name}/#{self.configuration.env}/mysql/#{timestamp}/#{File.basename(compressed_dump_file)}"
+    file = File.open(compressed_dump_file, 'r')
+
+    bucket = get_s3_bucket
+    obj = bucket.object(name)
+    obj.put(body: file)
+
+    file.close
+    File.unlink(compressed_dump_file)
+
+    obj
+  end
+
+  def self.clean_up_files(dump_file, compressed_dump_file)
+    self.log("Deleting #{dump_file}")
+    File.delete(dump_file) if File.exist?(dump_file)
+    self.log("Deleting #{compressed_dump_file}")
+    File.delete(compressed_dump_file) if File.exist?(compressed_dump_file)
+  end
+
+  def self.export_temp_db_to_file(dump_file, temp_db_config, temp_db)
+    self.log("Dumping temp DB to #{dump_file}")
+    system("mysqldump -h #{temp_db_config['host']} -u #{temp_db_config['username']} --password=#{temp_db_config['password']} #{temp_db} >> '#{dump_file}'")
+    if $?.exitstatus == 0
+      self.log("Dump created")
+    else
+      self.log("Failed to create dump")
+      return
+    end
+  end
+
+  def self.is_dev_or_integration_env?
+    self.configuration.env == 'development' || self.configuration.env == 'integration'
+  end
+
+  def self.sanitize_and_export_data
+    checks = self.pre_sanitization_checks
+    if checks[:pass]
+      dump_file, compressed_dump_file = self.create_files
+      self.clean_up_files(dump_file, compressed_dump_file)
+
+      # If in dev or integration env we don't need to sanatise the DB so we should
+      # just dump it to a file and upload
+      if self.is_dev_or_integration_env?
+        self.export_temp_db_to_file(dump_file, self.configuration.db_config, self.configuration.db_config["database"])
+      else
+        temp_db, temp_db_connection, temp_db_config = self.duplicate_database
+
+        self.sanitize_tables(temp_db_connection)
+
+        self.export_temp_db_to_file(dump_file, temp_db_config, temp_db)
+
+        self.clean_up_temp_db(temp_db)
+      end
+      self.gzip(dump_file)
+      self.upload(compressed_dump_file) if self.configuration.s3_bucket && self.configuration.aws_access_key_id && self.configuration.aws_secret_access_key
+      self.clean_up_files(dump_file, compressed_dump_file) unless self.configuration.s3_bucket && self.configuration.aws_access_key_id && self.configuration.aws_secret_access_key
+      self.log("-- DONE --")
+    else
+      self.log(checks[:error])
+    end
+  end
+
+  def self.import_data(env = nil, timestamp = nil)
+    env = "production" if env.nil?
+    prefix = "#{self.configuration.app_name}/#{env}/mysql"
+
+    bucket = get_s3_bucket
+    if timestamp.nil?
+      timestamp = bucket.objects(prefix: prefix).collect {|x| x.key[%r(#{prefix}\/(.*)\/), 1] }.max
+    end
+
+    # Check that there are files (as the user could have passed in an incorrect timestamp)
+    if timestamp.nil?
+      self.log("No mysql snapshot for timestamp #{prefix}/#{timestamp}")
+      return
+    end
+
+    self.log('WARNING: this rake task will dump your MySQL DB to tmp, then wipe your DB before importing a snapshot')
+    local_dump_file = "#{File.join(self.configuration.root, "tmp")}/local_data.dump"
+
+    # Make copy of local DB just in case something goes wrong
+    system("mysqldump -h #{self.configuration.db_config['host']} -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} #{self.configuration.db_config['database']} > '#{local_dump_file}'")
+    if $?.exitstatus == 0
+      self.log("Local DB dump stored in #{local_dump_file}")
+    else
+      raise "Failed to create a local DB dump. If a previous local dump exists, please delete it and try again."
+    end
+
+    # get all the files in the snapshot
+    objects = bucket.objects("#{prefix}/#{timestamp}")
+    dump_file = "#{File.join(self.configuration.root, "tmp")}/data.dump"
+    compressed_dump_file = "#{dump_file}.gz"
+    self.log("Downloading file to #{compressed_dump_file}")
+    url = objects.first.object.presigned_url(:get, expires_in: 600)
+    system("curl -o #{compressed_dump_file} '#{url}'")
+
+    # reset db
+    self.log("Recreating your local DB")
+    Rake::Task["db:drop"].invoke
+    Rake::Task["db:create"].invoke
+
+    # Import data
+    self.log("Unzipping and importing data...")
+    system("gunzip -c '#{compressed_dump_file}' | mysql -uroot #{self.configuration.db_config['database']}")
+    if $?.exitstatus == 0
+      File.delete(compressed_dump_file) if File.exist?(compressed_dump_file)
+    else
+      raise "Could not load #{compressed_dump_file} into DB #{self.configuration.db_config}"
+    end
+    self.log('-- DONE --')
+  end
+end

data/lib/tasks/rake_tasks.rb

@@ -0,0 +1,23 @@
+require 'rake'
+
+module ActiveSanitization
+  class RakeTasks
+    include Rake::DSL if defined? Rake::DSL
+
+    def install_tasks
+      namespace:active_sanitization do
+        desc "Sanitises MySQL database. If S3 creds are provided then the sanitized snapshot will be uploaded to S3"
+        task :sanitize_and_export_data => :environment do
+          ActiveSanitization.sanitize_and_export_data
+        end
+
+        desc "Import sanitized data from S3 into MySQL.  Optional arguments are `env` and `timestamp`.  These will default to 'production' and the latest snapshot if they are not provided"
+        task :import_data_from_s3, [:env, :timestamp] => [:environment] do |t, args|
+          ActiveSanitization.import_data(args[:env], args[:timestamp])
+        end
+      end
+    end
+  end
+end
+
+ActiveSanitization::RakeTasks.new.install_tasks

metadata

@@ -0,0 +1,200 @@
+--- !ruby/object:Gem::Specification
+name: active_sanitization
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Stephen Haley
+- Callum Dryden
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-04-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.1
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.10
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.10
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.1
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.18
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.18
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.4
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.33
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.33
+description: Active Santization provides any easy way to consistently sanitize data
+  from a MySql DB.  With the config you can specify how to sanitize each tables and
+  column.
+email:
+- stephen.haley@rightscale.com
+- callum.dryden@rightscale.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- active_sanitization.gemspec
+- lib/active_sanitization.rb
+- lib/active_sanitization/version.rb
+- lib/tasks/rake_tasks.rb
+homepage: https://github.com/rightscale/active_sanitization
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Quick in-place santization for your MySql DB
+test_files: []