mysql_truck 0.2.1 → 0.3.0

data/lib/mysql_truck/dumper.rb

@@ -27,27 +27,34 @@ module MysqlTruck
         if dump_table?(table)
 
           # This command creates a table_name.sql and a table_name.txt file
-          cmd = "mysqldump --quick -T #{tmp_path} "
-          cmd += csv_options
-          cmd += "#{db_connection_options} #{table}"
-          puts cmd
-          `#{cmd}`
-
-          # `mysqldump` creates files with .txt extensions, so we rename it.
-          mv filename(table)[:txt_file], filename(table)[:csv_file]
+          base_cmd = "mysqldump --quick"
+
+          # Dump schema
+          schema_cmd = "#{base_cmd} --no-data "
+          schema_cmd += "#{db_connection_options} #{table}"
+          schema_cmd += " > #{filename(table)[:schema_file]}"
+          puts schema_cmd
+          `#{schema_cmd}`
+
+          # Dump data
+          data_cmd = "#{base_cmd} --no-create-info "
+          data_cmd += "#{db_connection_options} #{table}"
+          data_cmd += " > #{filename(table)[:data_file]}"
+          puts data_cmd
+          `#{data_cmd}`
         end
 
         if split_schema_file?(table)
           schema_contents = filename(table)[:schema_file].read
 
           # Create schema with no indexes
-          File.open(filename(table)[:no_index_sql_file], 'w') do |f|
+          File.open(filename(table)[:no_index_schema_file], 'w') do |f|
             f.write(schema_contents.gsub(REGEX, ''))
           end
 
           # Create an alter table
           indices = []
-          File.open(filename(table)[:index_sql_file], 'w') do |f|
+          File.open(filename(table)[:alter_table_file], 'w') do |f|
             f.write("ALTER TABLE #{table}\n")
 
             schema_contents.gsub(/^,?\s*((UNIQUE)?\s*KEY\s`[A-Za-z\d_]+`\s*\([A-Za-z\d_,`]+\)),?\s*$/) do |part|
@@ -58,14 +65,14 @@ module MysqlTruck
         end
 
         if gzip_files?(table)
-          puts "gzipping #{filename(table)[:no_index_sql_file]}."
-          `gzip #{filename(table)[:no_index_sql_file]}`
+          puts "gzipping #{filename(table)[:no_index_schema_file]}."
+          `gzip #{filename(table)[:no_index_schema_file]}`
 
-          puts "gzipping #{filename(table)[:index_sql_file]}."
-          `gzip #{filename(table)[:index_sql_file]}`
+          puts "gzipping #{filename(table)[:alter_table_file]}."
+          `gzip #{filename(table)[:alter_table_file]}`
 
-          puts "gziping #{filename(table)[:csv_file]}."
-          `gzip #{filename(table)[:csv_file]}`
+          puts "gziping #{filename(table)[:data_file]}."
+          `gzip #{filename(table)[:data_file]}`
         end
 
         puts "#{table} dumped.\n\n"
@@ -105,28 +112,31 @@ module MysqlTruck
     end
 
     def bucket_path
-      @bucket_path ||= Pathname.new(bucket_dir).join(@time.strftime("%Y-%m-%d-%H-%M"))
+      @bucket_path ||= Pathname.new(bucket_dir).join(@time.strftime("%Y-%m-%d"))
     end
 
     def filename(table)
       @table_filenames ||= {}
       @table_filenames[table] ||= {
-        :schema_file          => tmp_path.join("#{table}.sql"),
-        :no_index_sql_file    => tmp_path.join("#{table}.no_index.sql"),
-        :index_sql_file       => tmp_path.join("#{table}.indices.sql"),
-        :txt_file             => tmp_path.join("#{table}.txt"),
-        :csv_file             => tmp_path.join("#{table}.csv"),
-        :gz_no_index_sql_file => tmp_path.join("#{table}.no_index.sql.gz"),
-        :gz_index_sql_file    => tmp_path.join("#{table}.indices.sql.gz"),
-        :gz_csv_file          => tmp_path.join("#{table}.csv.gz"),
+        # mysqldump files
+        :schema_file             => tmp_path.join("#{table}.schema.sql"),
+        :no_index_schema_file    => tmp_path.join("#{table}.no_index.sql"),
+
+        # altered mysql dump files
+        :alter_table_file        => tmp_path.join("#{table}.indices.sql"),
+        :data_file               => tmp_path.join("#{table}.data.sql"),
+
+        # Gzip filenames
+        :gz_no_index_schema_file => tmp_path.join("#{table}.no_index.sql.gz"),
+        :gz_alter_table_file     => tmp_path.join("#{table}.indices.sql.gz"),
+        :gz_data_file            => tmp_path.join("#{table}.data.sql.gz"),
       }
     end
 
     def gzip_files_exist?(table)
-      tmp_path.join("#{table}.sql.gz").file? &&
-      tmp_path.join("#{table}.no_index.sql.gz") &&
-      tmp_path.join("#{table}.indexes.sql.gz") &&
-      tmp_path.join("#{table}.csv.gz")
+      filename(table)[:gz_no_index_schema_file].file? &&
+      filename(table)[:gz_alter_table_file].file? &&
+      filename(table)[:gz_data_file].file?
     end
 
 
@@ -135,12 +145,7 @@ module MysqlTruck
       (
         smartly? &&
         !filename(table)[:schema_file].exist? &&
-        !filename(table)[:csv_file].exist? &&
-        !filename(table)[:no_index_sql_file].exist? &&
-        !filename(table)[:index_sql_file].exist? &&
-        !filename(table)[:gz_no_index_sql_file].exist? &&
-        !filename(table)[:gz_index_sql_file].exist? &&
-        !filename(table)[:gz_csv_file].exist?
+        !filename(table)[:data_file].exist?
       )
     end
 
@@ -149,12 +154,9 @@ module MysqlTruck
       (
         smartly? &&
         filename(table)[:schema_file].exist? &&
-        filename(table)[:csv_file].exist? &&
-        !filename(table)[:no_index_sql_file].exist? &&
-        !filename(table)[:index_sql_file].exist? &&
-        !filename(table)[:gz_no_index_sql_file].exist? &&
-        !filename(table)[:gz_index_sql_file].exist? &&
-        !filename(table)[:gz_csv_file].exist?
+        filename(table)[:data_file].exist? &&
+        !filename(table)[:no_index_schema_file].exist? &&
+        !filename(table)[:alter_table_file].exist?
       )
     end
 
@@ -162,13 +164,10 @@ module MysqlTruck
       !smartly? ||
       (
         smartly? &&
-        filename(table)[:schema_file].exist? &&
-        filename(table)[:csv_file].exist? &&
-        filename(table)[:no_index_sql_file].exist? &&
-        filename(table)[:index_sql_file].exist? &&
-        !filename(table)[:gz_no_index_sql_file].exist? &&
-        !filename(table)[:gz_index_sql_file].exist? &&
-        !filename(table)[:gz_csv_file].exist?
+        filename(table)[:data_file].exist? &&
+        filename(table)[:no_index_schema_file].exist? &&
+        filename(table)[:alter_table_file].exist? &&
+        !gzip_files_exist?(table)
       )
     end
   end # class Dumper

data/lib/mysql_truck/helper.rb

@@ -1,3 +1,4 @@
+require 'benchmark'
 module MysqlTruck
   module Helper
     include FileUtils
@@ -29,10 +30,6 @@ module MysqlTruck
       !local_host?
     end
 
-    def csv_options
-      " --fields-enclosed-by=\\\" --fields-terminated-by=, "
-    end
-
     def initialize_directories
       mkdir_p base_path
       mkdir_p tmp_path
@@ -59,5 +56,15 @@ module MysqlTruck
     def bucket_dir
       "mysql/#{config[:bucket_dir] || config[:database]}/"
     end
+
+    def benchmark
+      Benchmark.realtime do
+        yield
+      end
+    end
+
+    def formatted_time(time)
+      "#{"%0.5f" % time}ms"
+    end
   end
 end

data/lib/mysql_truck/loader.rb

@@ -25,7 +25,7 @@ module MysqlTruck
       @time = Time.new(*prefix.split("/").last.split("-"))
       initialize_directories
 
-      puts "Downloading backups"
+      puts "Download & gunzip backups"
       puts "-------------------"
       @bucket.keys(:prefix => prefix).each do |key|
         puts "\n#{key}"
@@ -34,8 +34,8 @@ module MysqlTruck
         # gunzip file
         if tmp_path.join(filename).exist?
           print " - Inflating #{filename} ... "
-          `gunzip -f #{tmp_path.join(filename)}`
-          print "complete.\n"
+          time = benchmark { `gunzip -f #{tmp_path.join(filename)}` }
+          print "complete (#{formatted_time time}).\n"
         end
       end
 
@@ -47,39 +47,48 @@ module MysqlTruck
       else
         import_cmd = "mysqlimport #{db_connection_options}"
       end
-      import_cmd += csv_options
+
+      files = Dir["#{tmp_path}/*.no_index.sql"]
+      total = files.size
+      count = 0
 
       # Find all .no_index.sql files and process
-      Dir["#{tmp_path}/*.no_index.sql"].each do |file|
-        table       = File.basename(file, ".no_index.sql")
-        puts "\nProcessing #{table}"
+      files.sort.each do |file|
+        count += 1
+        table = File.basename(file, ".no_index.sql")
+
+        puts "\nProcessing #{table} (#{count}/#{total})"
 
         schema_file = Pathname.new(file)
         index_file  = tmp_path.join("#{table}.indices.sql")
-        csv_file    = tmp_path.join("#{table}.csv")
+        data_file   = tmp_path.join("#{table}.data.sql")
 
 
         print " - Loading schema for #{table} ... "
         cmd = "cat #{schema_file} | mysql #{db_connection_options}"
-        `#{cmd}`
-        print "complete.\n"
+        time = benchmark { `#{cmd}` }
+        print "complete (#{formatted_time time}).\n"
 
-        if csv_file.exist?
+        if data_file.exist?
           print " - Importing #{schema_file.basename(".sql")} ... "
-          `#{import_cmd} #{csv_file}`
-          print "complete.\n"
+          time = benchmark do
+            `cat #{data_file} | mysql #{db_connection_options}`
+          end
+          print "complete (#{formatted_time time}).\n"
         end
 
         if index_file.exist?
           print " - Adding indices for #{schema_file.basename(".no_index.sql")} ... "
           cmd = "cat #{index_file} | mysql #{db_connection_options}"
-          `#{cmd}`
-          print "complete.\n"
+          time = benchmark do
+            `#{cmd}`
+          end
+          print "complete (#{formatted_time time}).\n"
         end
 
         schema_file.delete if schema_file.exist?
         index_file.delete if index_file.exist?
-        csv_file.delete if csv_file.exist?
+        data_file.delete if data_file.exist?
       end
 
       puts "Backup loaded."
@@ -106,13 +115,15 @@ module MysqlTruck
       if !smartly? || (smartly? && !unzipped_file.exist?)
         print " - Downloading... "
 
-        file.open("wb") do |f|
-          @bucket.s3.interface.get(@bucket.name, key.name) do |chunk|
-            f.write chunk
+        time = benchmark do
+          file.open("wb") do |f|
+            @bucket.s3.interface.get(@bucket.name, key.name) do |chunk|
+              f.write chunk
+            end
           end
         end
 
-        puts "complete."
+        puts "complete (#{formatted_time time})."
       else
         puts " already downloaded."
       end
@@ -133,7 +144,7 @@ module MysqlTruck
         return only_tables.include?(table_name)
       end
 
-      if filename.match(/\.csv\.gz$/)
+      if filename.match(/\.data\.sql\.gz$/)
         is_data = true
         is_schema = false
       else

data/lib/mysql_truck/version.rb

@@ -1,3 +1,3 @@
 module MysqlTruck
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mysql_truck
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -11,11 +11,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-30 00:00:00.000000000Z
+date: 2012-06-20 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: right_aws
-  requirement: &70252611012460 !ruby/object:Gem::Requirement
+  requirement: &70141600570020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -23,7 +23,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70252611012460
+  version_requirements: *70141600570020
 description: Mysql database backup tool. Dumps/Loads to/from S3.
 email:
 - peter@paydrotalks.com