dump_truck 0.0.2

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in dump_truck.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Allen Madsen
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,94 @@
+# DumpTruck
+
+A simple DSL to specify how to dump data from a production environment for use in a developer environment. Provides tools to limit what data is pulled and obfuscate data that is pulled.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'dump_truck'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install dump_truck
+
+## Usage
+
+Create a ruby file that looks like this:
+
+``` ruby
+require 'logger'
+
+DumpTruck.configure do
+  # You can optionally provide a logger. If no logger is provided,
+  # information will be logged to standard out.
+  logger Logger.new('dump.log')
+
+  # You can specify an arbitrary number of databases to connect to
+  database(:mysql) do
+    # Where and how to connect
+    hostname ENV['hostname']
+    user ENV['user']
+    password ENV['password'] # pass nil if there is no password
+
+    # The schemas to dump from this database.
+    schema 'app_production' do
+      # Directory to place the resulting sql dump
+      target_path ENV['target_path']
+
+      # File name for this schema sql dump. '.sql' is appended
+      # automatically to the name. If unspecified, the name of the schema
+      # is used.
+      target_file{|schema| schema + Time.now.strftime("_%Y%m%d%H%M%S")}
+
+      # Tables rules can be defined to have one of four modes:
+      # * keep - all data for that table is dumped
+      # * keep 'some query' - data that satifies the query is dumped
+      # * truncate - no data is dumped
+      # * ignore - the table and data are not dumped
+      #
+      # Tables also define obfuscation rules. If no rules is defined for
+      # a column, the value is dumped as is. Obfuscation rules can
+      # optionally receive the value and a number which will be unique
+      # for each row of that table. If an obfuscation rule is defined for
+      # a field that doesn't exist for the table, it is ignored. This
+      # is most useful for the default table rule, because it allows
+      # obfuscation of all email fields for tables that have them.
+      #
+      # The default table rule is applied to any table that does not have
+      # a rule defined. Otherwise, it behaves like any other table rule.
+
+      table_default do
+        keep "created_at > now() - interval 6 month"
+
+        obfuscate(:email){|email, n| "my.email+#{n}@gmail.com"}
+      end
+
+      table(:users) do
+        keep
+
+        obfuscate(:email){|email, n| "my.email+#{n}@gmail.com"}
+        obfuscate(:password){'password'}
+        obfuscate(:name){|name| name.split('').shuffle.join}
+      end
+
+      table(:delayed_jobs){keep "deleted_at is null"}
+      table(:roles){keep}
+      table(:emails){truncate}
+      table(:credit_cards){ignore}
+    end
+  end
+end
+```
+
+## Contributing
+
+1. Fork it ( http://github.com/secondrotation/dump_truck/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/dump_truck.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dump_truck/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "dump_truck"
+  spec.version       = DumpTruck::VERSION
+  spec.authors       = ["Allen Madsen"]
+  spec.email         = ["blatyo@gmail.com"]
+  spec.summary       = %q{DSL to dump production data for developer use.}
+  spec.description   = %q{DSL to dump production data for developer use. Allows selection of data and obfuscation.}
+  spec.homepage      = "https://github.com/secondrotation/dump_truck"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'colorize'
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry-debugger"
+end

data/lib/dump_truck.rb

@@ -0,0 +1,64 @@
+require 'dump_truck/version'
+
+require 'colorize'
+
+require 'dump_truck/configuration'
+require 'dump_truck/database_configuration'
+require 'dump_truck/schema_configuration'
+require 'dump_truck/table_configuration'
+
+require 'dump_truck/target'
+require 'dump_truck/truck'
+require 'dump_truck/loggable_truck'
+require 'dump_truck/mysql'
+
+class DumpTruck
+  attr_reader :config
+
+  def initialize(&block)
+    @config = Configuration.new(&block)
+  end
+
+  def dump
+    config.each_database do |db_config|
+      dump_database(db_config)
+    end
+  end
+
+private
+
+  def dump_database(db_config)
+    db_config.each_schema.map do |schema_config|
+      truck(db_config, schema_config)
+    end.map do |truck|
+      Thread.new(truck){|t| t.dump}
+    end.each do |thread|
+      thread.join
+    end
+  end
+
+  def truck(db_config, schema_config)
+    translator = translator_for(db_config.type)
+    client = client_for(db_config.type, db_config, schema_config)
+
+    LoggableTruck.new(schema_config, client, translator, config.logger)
+  end
+
+  def translator_for(type)
+    case type.to_sym
+    when :mysql
+      Mysql::Translator.new
+    else
+      raise "Unknown type #{type}"
+    end
+  end
+
+  def client_for(type, db_config, schema_config)
+    case type.to_sym
+    when :mysql
+      Mysql::Client.new(db_config, schema_config)
+    else
+      raise "Unknown type #{type}"
+    end
+  end
+end

data/lib/dump_truck/configuration.rb

@@ -0,0 +1,35 @@
+require 'logger'
+
+class DumpTruck
+  class Configuration
+    attr_reader :databases
+
+    def initialize
+      @databases = []
+      @logger = Logger.new($stdout)
+      @logger.formatter = proc do |severity, datetime, progname, msg|
+        "#{datetime.strftime('%Y-%m-%d %H:%M:%S').blue} #{msg}\n"
+      end
+
+      instance_eval(&Proc.new) if block_given?
+    end
+
+    def logger(logger = nil)
+      @logger = logger || @logger
+    end
+
+    def database(type, &block)
+      type = type.to_s.downcase.to_sym
+
+      @databases << DatabaseConfiguration.new(type, &block)
+    end
+
+    def each_database
+      if block_given?
+        @databases.each(&Proc.new)
+      else
+        @databases.each
+      end
+    end
+  end
+end

data/lib/dump_truck/database_configuration.rb

@@ -0,0 +1,61 @@
+class DumpTruck
+  class DatabaseConfiguration
+    attr_reader :type
+
+    def initialize(type)
+      @type = type
+      @schemas = {}
+
+      instance_eval(&Proc.new) if block_given?
+    end
+
+    def ssh_user(ssh_user = nil)
+      @ssh_user = ssh_user || @ssh_user
+    end
+
+    def ssh_hostname(ssh_hostname = nil)
+      @ssh_hostname = ssh_hostname || @ssh_hostname
+    end
+
+    def user(user = nil)
+      @user = user || @user
+    end
+
+    def password(password = nil)
+      @password = password || @password
+    end
+
+    def hostname(hostname = nil)
+      @hostname = hostname || @hostname
+    end
+
+    def schema(name, &block)
+      name = name.to_s
+      @schemas[name] = SchemaConfiguration.new(name, &block)
+    end
+
+    def schemas
+      @schemas.values
+    end
+
+    def each_schema
+      if block_given?
+        schemas.each(&Proc.new)
+      else
+        schemas.each
+      end
+    end
+
+    def ==(other)
+      type = other.type &&
+        user == other.user &&
+        password == other.password &&
+        hostname == other.hostname &&
+        schemas == other.schemas
+    end
+
+    def to_s
+      "<DumpTruck::DatabaseConfiguration(#{type}) (#{schemas.map(&:to_s).join(', ')})>"
+    end
+  end
+end

data/lib/dump_truck/loggable_truck.rb

@@ -0,0 +1,41 @@
+class DumpTruck
+  class LoggableTruck < Truck
+    def initialize(schema_config, client, translator, logger)
+      super(schema_config, client, translator)
+      @logger = logger
+      @processed_rows = 0
+    end
+
+  protected
+
+    def extract_tables
+      @logger.info("Getting tables from #{schema_config.name.blue}")
+      super
+    end
+
+    def extract_table(line)
+      table = super
+      @logger.info("=> #{table.green} in #{schema_config.name.blue}")
+      table
+    end
+
+    def dump_schema(tables)
+      @logger.info("Dumping data from #{schema_config.name.blue}")
+      super
+      @logger.info("Dumped data from #{schema_config.name.blue}")
+    end
+
+    def dump_data(config, table, target)
+      @logger.info("=> Dumping #{table.green} in #{schema_config.name.blue}")
+      super
+      @logger.info("=> Dumped #{@processed_rows.to_s.green} rows from #{table.green} in #{schema_config.name.blue}")
+      @processed_rows = 0
+    end
+
+    def extract_insert(line)
+      fields, data = super
+      @processed_rows += data.size
+      [fields, data]
+    end
+  end
+end

data/lib/dump_truck/mysql.rb

@@ -0,0 +1,7 @@
+class DumpTruck
+  module Mysql
+  end
+end
+
+require 'dump_truck/mysql/client'
+require 'dump_truck/mysql/translator'

data/lib/dump_truck/mysql/client.rb

@@ -0,0 +1,73 @@
+class DumpTruck
+  module Mysql
+    class Client
+      def initialize(connection_config, schema_config)
+        @ssh_user = connection_config.ssh_user
+        @ssh_hostname = connection_config.ssh_hostname
+        @schema = schema_config.name
+
+        write_defaults_file(connection_config)
+      end
+
+      def tables_dump
+        IO.popen(tables_dump_command, &Proc.new)
+      end
+
+      def data_dump(table_config, table)
+        IO.popen(data_dump_command(table_config, table), &Proc.new)
+      end
+
+    private
+      attr_reader :ssh_user, :ssh_hostname, :hostname, :user, :password, :schema, :file
+
+      def cnf_file
+        file.path
+      end
+
+      def write_defaults_file(connection_config)
+        hostname = connection_config.hostname
+        user = connection_config.user
+        password = connection_config.password
+
+        @file = Tempfile.new('my.cnf')
+        @file.write <<-MYCNF.gsub(/^\s+/, '')
+          [mysqldump]
+          user = #{user}
+          #{"password = #{password}" if !password.nil? && !password.empty?}
+          host = #{hostname}
+          hex-blob
+          no-create-db
+          skip-comments
+          single-transaction
+          complete-insert
+          compress
+          net_buffer_length = 1048576
+        MYCNF
+        @file.close
+      end
+
+      def tables_dump_command
+        "#{ssh_connection} mysqldump --defaults-extra-file=#{cnf_file} --no-data --no-set-names --no-tablespaces --skip-add-drop-table --skip-add-locks --skip-set-charset #{schema}"
+      end
+
+      def data_dump_command(config, table)
+        "#{ssh_connection} mysqldump --defaults-extra-file=#{cnf_file} #{data_flag(config)} #{where_flag(config)} #{schema} #{table}"
+      end
+
+      def ssh_connection
+        user_string = "#{@ssh_user}@" if @ssh_user
+
+        "ssh -C #{user_string}#{@ssh_hostname} " if @ssh_hostname
+      end
+
+      def data_flag(config)
+        config.mode == :none ? '--no-data' : ''
+      end
+
+      def where_flag(config)
+        config.mode == :some ? "--where='#{config.query}'" : ''
+      end
+    end
+  end
+end
+