dump_truck 0.0.2

data/lib/dump_truck/mysql/translator.rb

@@ -0,0 +1,123 @@
+class DumpTruck
+  module Mysql
+    class Translator
+      TABLE_NAME_REGEX = /^CREATE TABLE `(.*)`/
+      TABLE_FIELDS_REGEX = /^INSERT INTO `[^`]*` \((.*?)\)/
+      TABLE_VALUES_REGEX = /VALUES (.*);/
+      NUMBER = /-|\d/
+      INTEGER = /-?\d+/
+      FLOAT = /-?\d+\.\d+/
+      NON_QUOTE = /([^'\\]|\\.)*/
+
+      def table?(line)
+        line =~ TABLE_NAME_REGEX
+      end
+
+      def extract_table(line)
+        line.scan(TABLE_NAME_REGEX).flatten.first
+      end
+
+      def insert?(line)
+        line =~ TABLE_FIELDS_REGEX
+      end
+
+      def extract_insert(line)
+        fields = extract_fields(line)
+        [fields, extract_rows(line, fields)]
+      end
+
+      def generate_insert(table, keys, data)
+        "INSERT INTO `#{table}` (#{stringify_keys(keys)}) VALUES #{stringify_values(keys, data)};\n"
+      end
+
+    private
+
+      def extract_fields(line)
+        fields_string = line.scan(TABLE_FIELDS_REGEX).flatten.first
+        fields_string.split(', ').map{|field| field[1..-2]}
+      end
+
+      def extract_rows(line, fields)
+        values_string = line.scan(TABLE_VALUES_REGEX).flatten.first
+        data = extract_values(values_string, fields, line)
+        data.map{|values| Hash[fields.zip(values)]}
+      end
+
+      def extract_values(values_string, fields, line)
+        scanner = StringScanner.new(values_string)
+        data = []
+        values = []
+
+        until(scanner.eos?)
+          values << case scanner.peek(1)
+          when NUMBER
+            extract_number(scanner)
+          when "'"
+            extract_string(scanner)
+          when 'N'
+            extract_null(scanner)
+          when ')'
+            scanner.getch
+            data << values
+            values = []
+            next
+          when /[(, ]/
+            scanner.getch
+            next
+          else
+            context = scanner.pre_match[-60..-1] + scanner.peek(1) + scanner.rest[0..60]
+            raise "Unexpected character #{scanner.peek(1).inspect} at #{scanner.pos}. Last values: #{values.inspect}. Context: #{context.inspect}. Line: #{line.inspect}"
+          end
+        end
+
+        data
+      end
+
+      def extract_number(scanner)
+        if (number = scanner.scan(FLOAT))
+          number.to_f
+        else
+          scanner.scan(INTEGER).to_i
+        end
+      end
+
+      def extract_string(scanner)
+        scanner.getch
+        string = scanner.scan(NON_QUOTE)
+        scanner.getch
+
+        string
+      end
+
+      def extract_null(scanner)
+        scanner.scan(/NULL/)
+        nil
+      end
+
+      def stringify_keys(keys)
+        keys.map{|key| "`#{key}`"}.join(', ')
+      end
+
+      def stringify_values(keys, data)
+        data.map do |datum|
+          values = datum.values_at(*keys)
+          values_string = values.map{|value| stringify_field(value)}.join(',')
+          "(#{values_string})"
+        end.join(',')
+      end
+
+      def stringify_field(value)
+        case value
+        when nil
+          'NULL'
+        when Numeric
+          value.to_s
+        when String
+          "'#{value}'"
+        else
+          raise "Unable to stringify #{value.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/dump_truck/schema_configuration.rb

@@ -0,0 +1,55 @@
+class DumpTruck
+  class SchemaConfiguration
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+      @tables = Hash.new(TableConfiguration.new{truncate})
+      @target_path = 'tmp'
+      @target_name_generator = proc{|schema| schema}
+
+      instance_eval(&Proc.new) if block_given?
+    end
+
+    def target_path(target_path = nil)
+      @target_path = target_path || @target_path
+    end
+
+    def target_name
+      @target_name_generator = Proc.new
+    end
+
+    def target
+      File.join(@target_path, @target_name_generator.call(@name) + '.sql.gz')
+    end
+
+    def table_default
+      if block_given?
+        @tables.default = TableConfiguration.new(&Proc.new)
+      else
+        @tables.default
+      end
+    end
+
+    def table(name, &block)
+      name = name.to_s
+      @tables[name] = TableConfiguration.new(name, &block)
+    end
+
+    def tables
+      @tables.values
+    end
+
+    def table_config_for(table)
+      @tables[table.to_s]
+    end
+
+    def ==(other)
+      name == other.name && target_path == other.target_path && tables == other.tables
+    end
+
+    def to_s
+      "<DumpTruck::SchemaConfiguration(#{name}) target:#{target} (#{tables.map(&:to_s).join(', ')})>"
+    end
+  end
+end

data/lib/dump_truck/table_configuration.rb

@@ -0,0 +1,63 @@
+class DumpTruck
+  class TableConfiguration
+    attr_reader :name, :mode, :query, :obfuscators
+
+    def initialize(name = nil)
+      @name = name.to_s
+      @obfuscators = Hash.new(proc{|value| value})
+      @mode = :none
+
+      instance_eval(&Proc.new) if block_given?
+    end
+
+    def truncate
+      @mode = :none
+    end
+
+    def ignore
+      @mode = :ignore
+    end
+
+    def keep(query = nil)
+      @query = query
+
+      if query
+        @mode = :some
+      else
+        @mode = :all
+      end
+    end
+
+    def obfuscate_default
+      if block_given?
+        @obfuscators.default = Proc.new
+      else
+        @obfuscators.default
+      end
+    end
+
+    def obfuscate(column)
+      column = column.to_s
+      if block_given?
+        @obfuscators[column] = Proc.new
+      else
+        @obfuscators[column] = proc{nil}
+      end
+    end
+
+    def obfuscate_value(column, value, n)
+      @obfuscators[column.to_s].call(value, n)
+    end
+
+    def ==(other)
+      name == other.name &&
+        mode == other.mode &&
+        (mode != :some || query == other.query)
+    end
+
+    def to_s
+      query_string = mode == :some ? ", query: #{query}" : ''
+      "<DumpTruck::TableConfiguration(#{name}) mode: #{mode}#{query_string} (#{obfuscators.keys.join(', ')})>"
+    end
+  end
+end

data/lib/dump_truck/target.rb

@@ -0,0 +1,13 @@
+class DumpTruck
+  class Target
+    def initialize(target_file)
+      @target_file = target_file
+    end
+
+    def open
+      Zlib::GzipWriter.open(@target_file) do |target|
+        yield target
+      end
+    end
+  end
+end

data/lib/dump_truck/truck.rb

@@ -0,0 +1,70 @@
+require 'zlib'
+
+class DumpTruck
+  class Truck
+    def initialize(schema_config, client, translator)
+      @schema_config = schema_config
+      @client = client
+      @translator = translator
+    end
+
+    def dump
+      tables = extract_tables
+      dump_schema(tables)
+    end
+
+  protected
+
+    attr_reader :schema_config, :client, :translator
+
+    def extract_tables
+      client.tables_dump do |io|
+        io.each.map do |line|
+          extract_table(line) if translator.table?(line)
+        end.compact
+      end
+    end
+
+    def dump_schema(tables)
+      Target.new(schema_config.target).open do |target|
+        tables.each do |table|
+          config = schema_config.table_config_for(table)
+
+          dump_data(config, table, target) unless config.mode == :ignore
+        end
+      end
+    end
+
+    def dump_data(config, table, target)
+      client.data_dump(config, table) do |io|
+        row_num = 0
+        io.each do |line|
+          line, row_num = translator.insert?(line) ? obfuscate(config, table, line, row_num) : [line, row_num]
+
+          target.write(line)
+        end
+      end
+    end
+
+    def obfuscate(config, table, line, row_num)
+      return '' if config.mode == :none
+
+      fields, data = extract_insert(line)
+
+      data = data.map do |datum|
+        row_num += 1
+        Hash[datum.map{|field, value| [field, config.obfuscate_value(field, value, row_num)]}]
+      end
+
+      [translator.generate_insert(table, fields, data), row_num]
+    end
+
+    def extract_table(line)
+      translator.extract_table(line)
+    end
+
+    def extract_insert(line)
+      translator.extract_insert(line)
+    end
+  end
+end

data/lib/dump_truck/version.rb

@@ -0,0 +1,3 @@
+class DumpTruck
+  VERSION = "0.0.2"
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe DumpTruck::Configuration do
+  describe '.new' do
+    context "when a configuration block is passed" do
+      let(:config) do
+        DumpTruck::Configuration.new do
+          database(:mysql){}
+          database(:postgres){}
+        end
+      end
+      subject{config}
+
+      its(:databases) do
+        should eq([
+          DumpTruck::DatabaseConfiguration.new(:mysql),
+          DumpTruck::DatabaseConfiguration.new(:postgres)
+        ])
+      end
+    end
+  end
+end

data/spec/databases_configuration_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe DumpTruck::DatabaseConfiguration do
+  describe '.new' do
+    context "when a configuration block is passed" do
+      let(:config) do
+        DumpTruck::DatabaseConfiguration.new(:mysql) do
+          hostname 'localhost'
+          user 'root'
+          password 'secret'
+
+          schema('app_production'){}
+          schema('app_staging'){}
+        end
+      end
+      subject{config}
+
+      its(:hostname){should eq('localhost')}
+      its(:user){should eq('root')}
+      its(:password){should eq('secret')}
+      its(:schemas) do
+        should eq([
+          DumpTruck::SchemaConfiguration.new('app_production'),
+          DumpTruck::SchemaConfiguration.new('app_staging')
+        ])
+      end
+    end
+  end
+end

data/spec/dump_truck_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe DumpTruck do
+  it 'should have a version number' do
+    expect(DumpTruck::VERSION).not_to be nil
+  end
+
+  describe '.new' do
+    context "when given the config for a database" do
+      let(:truck_config) do
+        DumpTruck.new do
+          database(:mysql){}
+        end.config
+      end
+      subject{truck_config}
+
+      its(:databases) do
+        should eq([DumpTruck::DatabaseConfiguration.new(:mysql)])
+      end
+    end
+  end
+
+  describe '#dump' do
+    let(:truck) do
+      DumpTruck.new do
+        database(:mysql) do
+          user 'root'
+          hostname 'localhost'
+
+          schema(:app_production) do
+            target_path(File.expand_path('../../tmp', __FILE__))
+
+            table(:roles){keep}
+          end
+        end
+      end
+    end
+  end
+end

data/spec/fixtures/mysql/expected_dump.sql

@@ -0,0 +1,147 @@
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+DROP TABLE IF EXISTS `permissions`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `permissions` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL,
+  `created_at` datetime DEFAULT NULL,
+  `updated_at` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `index_permissions_on_name` (`name`)
+) ENGINE=InnoDB AUTO_INCREMENT=343 DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+LOCK TABLES `permissions` WRITE;
+/*!40000 ALTER TABLE `permissions` DISABLE KEYS */;
+INSERT INTO `permissions` (`id`, `name`, `created_at`, `updated_at`) VALUES (12,'account:login','2009-08-11 11:10:17','2009-08-11 11:10:17'),(24,'job_artifacts:view','2009-08-11 11:10:17','2009-08-11 11:10:17'),(36,'charities:view','2009-08-11 11:10:17','2009-08-11 11:10:17'),(48,'charities:create','2009-08-11 11:10:17','2009-08-11 11:10:17'),(6,'charities:edit','2009-08-11 11:10:17','2009-08-11 11:10:17'),(61,'careers:create','2009-08-11 11:10:17','2009-08-11 11:10:17'),(68,'careers:view','2009-08-11 11:10:18','2009-08-11 11:10:18'),(90,'careers:edit','2009-08-11 11:10:18','2009-08-11 11:10:18'),(10,'media_stories:create','2009-08-11 11:10:18','2009-08-11 11:10:18'),(101,'media_stories:view','2009-08-11 11:10:18','2009-08-11 11:10:18'),(113,'media_stories:edit','2009-08-11 11:10:18','2009-08-11 11:10:18'),(123,'media_stories:destroy','2009-08-11 11:10:18','2009-08-11 11:10:18');
+/*!40000 ALTER TABLE `permissions` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+DROP TABLE IF EXISTS `roles`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `roles` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL,
+  `description` varchar(255) DEFAULT NULL,
+  `created_at` datetime DEFAULT NULL,
+  `updated_at` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `index_roles_on_name` (`name`)
+) ENGINE=InnoDB AUTO_INCREMENT=94 DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+LOCK TABLES `roles` WRITE;
+/*!40000 ALTER TABLE `roles` DISABLE KEYS */;
+INSERT INTO `roles` (`id`, `name`, `description`, `created_at`, `updated_at`) VALUES (21,'1',NULL,'2009-08-11 11:10:17','2009-08-11 11:10:17'),(23,'2',NULL,'2009-08-11 11:10:17','2009-08-11 11:10:17'),(34,'3',NULL,'2009-08-11 11:10:17','2009-08-11 11:10:17'),(84,'4',NULL,'2009-08-11 11:10:18','2009-08-11 11:10:18'),(85,'5',NULL,'2009-08-11 11:10:18','2009-08-11 11:10:18'),(96,'6',NULL,'2009-08-11 11:10:18','2009-08-11 11:10:18'),(207,'7',NULL,'2009-08-11 11:10:18','2009-08-11 11:10:18'),(308,'8',NULL,'2009-08-11 11:10:18','2009-08-11 11:10:18');
+/*!40000 ALTER TABLE `roles` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+DROP TABLE IF EXISTS `users`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `users` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `login` varchar(255) CHARACTER SET utf8 NOT NULL,
+  `email` varchar(255) CHARACTER SET utf8 NOT NULL,
+  `encrypted_password` varchar(128) NOT NULL DEFAULT '',
+  `password_salt` varchar(128) NOT NULL DEFAULT '',
+  `created_at` datetime DEFAULT NULL,
+  `updated_at` datetime DEFAULT NULL,
+  `remember_token` varchar(255) CHARACTER SET utf8 DEFAULT NULL,
+  `remember_token_expires_at` datetime DEFAULT NULL,
+  `confirmed_agreement` tinyint(1) NOT NULL DEFAULT '0',
+  `first_name` varchar(255) CHARACTER SET utf8 DEFAULT NULL,
+  `middle_name` varchar(255) CHARACTER SET utf8 DEFAULT NULL,
+  `last_name` varchar(255) CHARACTER SET utf8 DEFAULT NULL,
+  `role` int(11) DEFAULT '0',
+  `status` int(11) DEFAULT '0',
+  `pw_reset_code` varchar(40) CHARACTER SET utf8 DEFAULT NULL,
+  `activated` tinyint(1) DEFAULT '0',
+  `activation_key` varchar(40) DEFAULT NULL,
+  `telephone` varchar(255) DEFAULT NULL,
+  `confirmed_referral_terms` tinyint(1) DEFAULT '0',
+  `confirmed_referral_terms_at` datetime DEFAULT NULL,
+  `current_facility_id` int(11) DEFAULT NULL,
+  `selected_drive_id` int(11) DEFAULT NULL,
+  `triggers_qc` tinyint(1) NOT NULL DEFAULT '0',
+  `persistence_token` varchar(255) DEFAULT NULL,
+  `single_access_token` varchar(255) DEFAULT NULL,
+  `requires_ldap_authentication` tinyint(1) DEFAULT '0',
+  `sign_in_count` int(11) DEFAULT '0',
+  `current_sign_in_at` datetime DEFAULT NULL,
+  `last_sign_in_at` datetime DEFAULT NULL,
+  `current_sign_in_ip` varchar(255) DEFAULT NULL,
+  `last_sign_in_ip` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `index_users_on_email` (`email`),
+  UNIQUE KEY `index_users_on_single_access_token` (`single_access_token`),
+  UNIQUE KEY `index_users_on_pw_reset_code` (`pw_reset_code`),
+  KEY `index_users_on_activation_key` (`activation_key`),
+  KEY `index_users_on_activated` (`activated`),
+  KEY `index_users_on_last_name` (`last_name`)
+) ENGINE=InnoDB AUTO_INCREMENT=919492 DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+LOCK TABLES `users` WRITE;
+/*!40000 ALTER TABLE `users` DISABLE KEYS */;
+/*!40000 ALTER TABLE `users` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+