mysql_truck 0.6.4 → 0.6.6

data/Gemfile

@@ -2,3 +2,9 @@ source "http://rubygems.org"
 
 # Specify your gem's dependencies in mysql_truck.gemspec
 gemspec
+
+group :test do
+  gem 'minitest'
+  gem 'mocha'
+  gem 'hashie'
+end

data/bin/mysql_truck

@@ -73,6 +73,10 @@ parser = OptionParser.new do |opts|
     options[:skip_data_for_tables] = tables.split(",")
   end
 
+  opts.on("--skip-all-data", "Only load schema") do
+    options[:skip_all_data] = true
+  end
+
   opts.on("-o", "--only-tables TABLES",
           "List of tables to dump/load.") do |tables|
     options[:only_tables] = tables.split(",")

data/lib/mysql_truck/dumper.rb

@@ -49,7 +49,7 @@ module MysqlTruck
           # Dump data
           puts "Writing data for #{table} ..."
           benchmark do
-            puts `mysql #{db_connection_options} --skip-column-names --batch -e "select * from #{table}" | lzop -6 > #{filename(table)[:compressed_data_file]} 2>/var/log/mysql_truck_dump.log`
+            puts `mysql #{db_connection_options} --skip-column-names --batch -e "select * from #{table}" | sed -e 's/\tNULL/\t\\\\N/g'| lzop -6 > #{filename(table)[:compressed_data_file]} 2>/var/log/mysql_truck_dump.log`
             if $?.exitstatus != 0
               puts "Command did not execute successfully"
             end

data/lib/mysql_truck/loader.rb

@@ -19,7 +19,6 @@ module MysqlTruck
     end
 
     def load_backup
-
       # Default to latest backup
       backup_date_str = config[:backup_date] || backups.first.split("/").last
 
@@ -30,7 +29,6 @@ module MysqlTruck
       puts "Download & decompressing backups"
       puts "-------------------"
 
-
       @bucket.keys(:prefix => s3_path).each do |key|
         next if key.to_s.match(/\/$/)
         next unless (filename = download_file(key))
@@ -68,7 +66,7 @@ module MysqlTruck
         csv_data_file   = tmp_path.join("#{table}.csv")
         tsv_data_file   = tmp_path.join("#{table}.data.tsv")
 
-
+        # Create table
         if schema_file.exist?
           print " - Loading schema: #{File.basename(schema_file)} ... "
           execute_sql_file(table, backup_date_str, schema_file)
@@ -84,23 +82,26 @@ module MysqlTruck
           next
         end
 
-
-        if data_file.exist?
-          print " - Importing #{File.basename(data_file)} ... "
-          execute_sql_file(table, backup_date_str, data_file)
-          data_file.delete
-
-        elsif csv_data_file.exist?
-          print " - Importing #{File.basename(csv_data_file)} ... "
-          csv_data_file = import_csv_file(table, backup_date_str, csv_data_file)
-          File.delete(csv_data_file)
-
-        elsif tsv_data_file.exist?
-          print " - Importing #{File.basename(tsv_data_file)} ... "
-          tsv_data_file = import_tsv_file(table, backup_date_str, tsv_data_file)
-          File.delete(tsv_data_file)
+        # Load data
+        if !config[:skip_all_data]
+          if data_file.exist?
+            print " - Importing #{File.basename(data_file)} ... "
+            execute_sql_file(table, backup_date_str, data_file)
+            data_file.delete
+
+          elsif csv_data_file.exist?
+            print " - Importing #{File.basename(csv_data_file)} ... "
+            csv_data_file = import_csv_file(table, backup_date_str, csv_data_file)
+            File.delete(csv_data_file)
+
+          elsif tsv_data_file.exist?
+            print " - Importing #{File.basename(tsv_data_file)} ... "
+            tsv_data_file = import_tsv_file(table, backup_date_str, tsv_data_file)
+            File.delete(tsv_data_file)
+          end
         end
 
+        # Add indices
         if index_file.exist?
           print " - Adding indices: #{File.basename(index_file)} ... "
           execute_sql_file(table, backup_date_str, index_file)
@@ -191,8 +192,7 @@ module MysqlTruck
 
     def should_download_file?(filename)
       table_name = filename.gsub(/\..*\..*$/, '')
-
-      if only_tables.empty? and skip_data_for_tables.empty?
+      if only_tables.empty? and skip_data_for_tables.empty? and !config[:skip_all_data]
         return true
       end
 
@@ -202,7 +202,8 @@ module MysqlTruck
         return only_tables.include?(table_name)
       end
 
-      if filename.match(/\.data\.sql\.gz$/)
+      if filename.match(/\.data\.(sql|tsv)\.(lzo|gz)$/)
+        return false if config[:skip_all_data]
         is_data = true
         is_schema = false
       else
@@ -210,8 +211,10 @@ module MysqlTruck
         is_schema = true
       end
 
+      return true if is_schema && config[:skip_all_data]
+
       if !skip_data_for_tables.empty?
-        if is_schema or (is_data and !skip_data_for_tables.include?(table_name))
+        if is_schema || (is_data && !skip_data_for_tables.include?(table_name))
           return true
         end
       end
@@ -237,6 +240,7 @@ module MysqlTruck
         end
         @backups = @backups.sort { |a,b| b <=> a }
       end
+
       @backups
     end
 

data/lib/mysql_truck/version.rb

@@ -1,3 +1,3 @@
 module MysqlTruck
-  VERSION = "0.6.4"
+  VERSION = "0.6.6"
 end

data/test/fake_aws.rb

@@ -0,0 +1,69 @@
+require 'hashie'
+
+class FakeAws
+
+  def initialize
+    @files = []
+  end
+
+  def bucket(bucket_name)
+    BucketContent.new(bucket_name, @files)
+  end
+
+  def add_fake_file(path)
+    @files << path
+  end
+
+end
+
+class BucketContent
+
+  def initialize(name, data)
+    @data = data
+    @name = name
+  end
+
+  def name
+    @name
+  end
+
+  def s3
+    FakeS3.new(@data)
+  end
+
+  def keys(configs)
+    prefix = configs[:prefix]
+    @data.map do |file|
+      Hashie::Mash.new({name: file})
+    end
+  end
+end
+
+
+class FakeS3
+
+  def initialize(data)
+    @data = data
+  end
+
+  def interface
+    #TODO calculate common_prefixes
+    Interface.new(@data)
+  end
+end
+
+class Interface
+
+  def initialize(data)
+    @data = data
+  end
+
+  def incrementally_list_bucket(bucket_name, options)
+   yield  common_prefixes: [@data.first.split('/')[0..2].join('/') ]
+  end
+
+  def get(bucket_name, key_name)
+    yield 'content'
+  end
+
+end

data/test/lib/mysql_truck/loader_test.rb

@@ -0,0 +1,82 @@
+require 'test_helper'
+
+module MysqlTruck
+  class LoaderTest < Minitest::Unit::TestCase
+
+    def setup
+      mock_aws_with_fake_backups('2013-01-01', ['users', 'mixes'])
+    end
+
+    def test_loads_old_backups_with_only_tables
+      config = { s3_access_key: 'key', s3_secret_access_key: 'password', :only_tables => ['users'] }
+      loader = MysqlTruck::Loader.new(config)
+
+      assert loader.should_download_file?('users.data.sql.gz')
+      assert loader.should_download_file?('users.indices.sql.gz')
+      assert loader.should_download_file?('users.no_index.sql.gz')
+
+
+      refute loader.should_download_file?('mixes.data.sql.gz')
+      refute loader.should_download_file?('mixes.indices.sql.gz')
+      refute loader.should_download_file?('mixes.no_index.sql.gz')
+    end
+
+    def test_old_backups_with_skip_data_for_tables
+      config = { s3_access_key: 'key', s3_secret_access_key: 'password', :skip_data_for_tables => ['users'] }
+      loader = MysqlTruck::Loader.new(config)
+
+
+      #refutes data files for the ones you want to ignore
+      refute loader.should_download_file?('users.data.sql.gz')
+      assert loader.should_download_file?('users.indices.sql.gz')
+      assert loader.should_download_file?('users.no_index.sql.gz')
+
+      assert loader.should_download_file?('mixes.data.sql.gz')
+      assert loader.should_download_file?('mixes.indices.sql.gz')
+      assert loader.should_download_file?('mixes.no_index.sql.gz')
+    end
+
+    def test_new_backups_with_only_tables
+      config = { s3_access_key: 'key', s3_secret_access_key: 'password', :only_tables => ['users'] }
+      loader = MysqlTruck::Loader.new(config)
+
+      assert loader.should_download_file?('users.data.tsv.lzo')
+      assert loader.should_download_file?('users.indices.sql.lzo')
+      assert loader.should_download_file?('users.no_index.sql.lzo')
+
+
+      refute loader.should_download_file?('mixes.data.tsv.lzo')
+      refute loader.should_download_file?('mixes.indices.sql.lzo')
+      refute loader.should_download_file?('mixes.no_index.sql.lzo')
+    end
+
+    def test_new_backups_with_with_skip_data_for_tables
+      config = { s3_access_key: 'key', s3_secret_access_key: 'password', :skip_data_for_tables => ['users'] }
+      loader = MysqlTruck::Loader.new(config)
+
+      #refutes data files for the ones you want to ignore
+      refute loader.should_download_file?('users.data.tsv.lzo')
+      assert loader.should_download_file?('users.indices.sql.lzo')
+      assert loader.should_download_file?('users.no_index.sql.lzo')
+
+      assert loader.should_download_file?('mixes.data.tsv.lzo')
+      assert loader.should_download_file?('mixes.indices.sql.lzo')
+      assert loader.should_download_file?('mixes.no_index.sql.lzo')
+    end
+
+    def mock_aws_with_fake_backups(date, tables)
+      fake_aws = FakeAws.new
+
+      tables.each do |table|
+        base_name = "mysql/8trackscom/#{date}/#{table}"
+
+        fake_aws.add_fake_file("#{base_name}.data.sql.gz")
+        fake_aws.add_fake_file("#{base_name}.indices.sql.gz")
+        fake_aws.add_fake_file("#{base_name}.no_index.sql.gz")
+      end
+
+      RightAws::S3.expects(:new).returns(fake_aws)
+    end
+
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require 'minitest/autorun'
+require 'mocha/setup'
+require 'fake_aws'
+require_relative '../lib/mysql_truck'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mysql_truck
 version: !ruby/object:Gem::Version
-  version: 0.6.4
+  version: 0.6.6
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-04 00:00:00.000000000 Z
+date: 2013-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: right_aws
@@ -66,6 +66,9 @@ files:
 - lib/mysql_truck/loader.rb
 - lib/mysql_truck/version.rb
 - mysql_truck.gemspec
+- test/fake_aws.rb
+- test/lib/mysql_truck/loader_test.rb
+- test/test_helper.rb
 homepage: ''
 licenses: []
 post_install_message: 
@@ -90,4 +93,7 @@ rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Mysql database backup tool. Dumps/Loads to/from S3.
-test_files: []
+test_files:
+- test/fake_aws.rb
+- test/lib/mysql_truck/loader_test.rb
+- test/test_helper.rb