mysql_truck 0.5.7.1 → 0.6.0

data/.gitignore

@@ -2,3 +2,4 @@
 .bundle
 Gemfile.lock
 pkg/*
+tags

data/bin/mysql_truck

@@ -64,11 +64,20 @@ parser = OptionParser.new do |opts|
     options[:dump_dir] = dir
   end
 
+  opts.on("--grantees GRANTEE", "List of comma separated S3 account ids that can access this data") do |g|
+    options[:grantees] = g
+  end
+
   opts.on("-t", "--skip-tables TABLES",
           "List of tables to skip separated by commas.") do |tables|
     options[:skip_data_for_tables] = tables.split(",")
   end
 
+  opts.on("-o", "--only-tables TABLES",
+          "List of tables to dump/load.") do |tables|
+    options[:only_tables] = tables.split(",")
+  end
+
   opts.on("-e", "--exec-smartly",
           "On dumping, do not dump tables that have already been dumped. On loading, if the files were already downloaded, do not redownload. This option allows for resuming a previous load/dump that failed.") do
     options[:smartly] = true

data/lib/mysql_truck/dumper.rb

@@ -22,11 +22,9 @@ module MysqlTruck
     def dump_data
       tables.each do |table|
         puts "Dumping #{table}..."
-        next if gzip_files_exist?(table) && smartly?
+        next if compressed_files_exist?(table) && smartly?
 
         if dump_table?(table)
-
-          # This command creates a table_name.sql and a table_name.txt file
           base_cmd = "mysqldump --quick"
 
           # Dump schema
@@ -37,11 +35,23 @@ module MysqlTruck
           `#{schema_cmd}`
 
           # Dump data
-          data_cmd = "#{base_cmd} --no-create-info "
-          data_cmd += "#{db_connection_options} #{table}"
-          data_cmd += " > #{filename(table)[:data_file]}"
-          puts data_cmd
-          `#{data_cmd}`
+          puts "Writing data for #{table} ..."
+          mysql = Mysql.init
+          mysql.options(Mysql::SET_CHARSET_NAME, "utf8")
+          mysql.connect(
+            config[:host],
+            config[:username],
+            config[:password],
+            config[:database],
+            config[:port]
+          )
+          mysql.query("SET NAMES utf8")
+          CSV.open(filename(table)[:data_file], 'wb', :col_sep => "\t") do |csv|
+            result = mysql.query("SELECT * FROM #{table}")
+            result.each do |row|
+              csv << row.collect { |col| col && col.match(/\t/) ? "    " : col }
+            end
+          end
         end
 
         if split_schema_file?(table)
@@ -64,15 +74,17 @@ module MysqlTruck
           end
         end
 
-        if gzip_files?(table)
-          puts "gzipping #{filename(table)[:no_index_schema_file]}."
-          `gzip #{filename(table)[:no_index_schema_file]}`
+        if compress_files?(table)
+          puts "compressing #{filename(table)[:no_index_schema_file]}."
 
-          puts "gzipping #{filename(table)[:alter_table_file]}."
-          `gzip #{filename(table)[:alter_table_file]}`
+          compression_cmd = "lzop -8 -U"
+          `#{compression_cmd} #{filename(table)[:no_index_schema_file]}`
 
-          puts "gziping #{filename(table)[:data_file]}."
-          `gzip #{filename(table)[:data_file]}`
+          puts "compressing #{filename(table)[:alter_table_file]}."
+          `#{compression_cmd} #{filename(table)[:alter_table_file]}`
+
+          puts "compressing #{filename(table)[:data_file]}."
+          `#{compression_cmd} #{filename(table)[:data_file]}`
         end
 
         puts "#{table} dumped.\n\n"
@@ -81,7 +93,7 @@ module MysqlTruck
 
     def upload
       Dir["#{tmp_path}/*"].each do |file|
-        next if File.extname(file) != ".gz"
+        next if File.extname(file) != ".lzo"
         puts "Uploading #{file} ..."
         upload_file file
       end
@@ -96,10 +108,31 @@ module MysqlTruck
 
     def upload_file(local_file)
       path = Pathname.new(local_file)
-      s3_path = bucket_path.join(path.basename)
+
+      # Store all schema files in the <DATE>/schema/ dir
+      if local_file.match(/\.indices\.sql/) || local_file.match(/\.no_index\.sql/)
+        s3_path = bucket_path.join("_schema", path.basename)
+
+      # Store data files in <DATE>/<TABLE>/<TABLE>.data.tsv.lzo
+      elsif local_file.match(/\.data\.tsv/)
+        table_name = File.basename(local_file).match(/(.*)\.data\.tsv\.lzo/)[1]
+        s3_path = bucket_path.join(table_name, path.basename)
+  puts "Path: #{s3_path}"
+
+      else
+        puts "** Couldn't upload #{local_file}!!"
+        puts "** Couldn't upload #{local_file}!!"
+        return
+      end
+puts s3_path
       @bucket.put(s3_path, open(path), {}, nil, {
         'x-amz-storage-class' => 'REDUCED_REDUNDANCY'
       })
+
+      key = RightAws::S3::Key.create(@bucket, s3_path)
+      config[:grantees].split(",").each do |grantee|
+        RightAws::S3::Grantee.new(key, grantee, "FULL_CONTROL", :apply)
+      end
     end
 
     def tables
@@ -124,19 +157,21 @@ module MysqlTruck
 
         # altered mysql dump files
         :alter_table_file        => tmp_path.join("#{table}.indices.sql"),
-        :data_file               => tmp_path.join("#{table}.data.sql"),
 
-        # Gzip filenames
-        :gz_no_index_schema_file => tmp_path.join("#{table}.no_index.sql.gz"),
-        :gz_alter_table_file     => tmp_path.join("#{table}.indices.sql.gz"),
-        :gz_data_file            => tmp_path.join("#{table}.data.sql.gz"),
+        # Tab seperated data
+        :data_file               => tmp_path.join("#{table}.data.tsv"),
+
+        # compressed filenames
+        :compressed_no_index_schema_file => tmp_path.join("#{table}.no_index.sql.lzo"),
+        :compressed_alter_table_file     => tmp_path.join("#{table}.indices.sql.lzo"),
+        :compressed_data_file            => tmp_path.join("#{table}.data.tsv.lzo"),
       }
     end
 
-    def gzip_files_exist?(table)
-      filename(table)[:gz_no_index_schema_file].file? &&
-      filename(table)[:gz_alter_table_file].file? &&
-      filename(table)[:gz_data_file].file?
+    def compressed_files_exist?(table)
+      filename(table)[:compressed_no_index_schema_file].file? &&
+      filename(table)[:compressed_alter_table_file].file? &&
+      filename(table)[:compressed_data_file].file?
     end
 
 
@@ -160,14 +195,14 @@ module MysqlTruck
       )
     end
 
-    def gzip_files?(table)
+    def compress_files?(table)
       !smartly? ||
       (
         smartly? &&
         filename(table)[:data_file].exist? &&
         filename(table)[:no_index_schema_file].exist? &&
         filename(table)[:alter_table_file].exist? &&
-        !gzip_files_exist?(table)
+        !compressed_files_exist?(table)
       )
     end
   end # class Dumper

data/lib/mysql_truck/helper.rb

@@ -18,6 +18,7 @@ module MysqlTruck
       opts = %Q[ -u #{config[:username]} ]
       opts += %Q[ -p"#{config[:password]}" ] unless config[:password].nil?
       opts += %Q[ -h #{config[:host]} --default-character-set=utf8 ]
+      opts += %Q[ -P #{config[:port]} ] if config[:port]
       opts += %Q[ #{config[:database]} ]
       opts
     end

data/lib/mysql_truck/loader.rb

@@ -27,17 +27,23 @@ module MysqlTruck
       @time = Time.new(*backup_date_str.split("-"))
       initialize_directories
 
-      puts "Download & gunzip backups"
+      puts "Download & decompressing backups"
       puts "-------------------"
 
 
       @bucket.keys(:prefix => s3_path).each do |key|
+        next if key.to_s.match(/\/$/)
         next unless (filename = download_file(key))
 
-        # gunzip file
         if tmp_path.join(filename).exist?
           print " - Inflating #{filename} ... "
-          time = benchmark { `gunzip -f #{tmp_path.join(filename)}` }
+
+          if File.extname(filename).match(".lzo")
+            decompress_cmd = "lzop -d -f -U"
+          else
+            decompress_cmd = "gunzip -f"
+          end
+          time = benchmark { `#{decompress_cmd} #{tmp_path.join(filename)}` }
           print "complete (#{formatted_time time}).\n"
         end
       end
@@ -61,6 +67,7 @@ module MysqlTruck
         index_file      = tmp_path.join("#{table}.indices.sql")
         data_file       = tmp_path.join("#{table}.data.sql")
         csv_data_file   = tmp_path.join("#{table}.csv")
+        tsv_data_file   = tmp_path.join("#{table}.data.tsv")
 
 
         if schema_file.exist?
@@ -88,6 +95,11 @@ module MysqlTruck
           print " - Importing #{File.basename(csv_data_file)} ... "
           csv_data_file = import_csv_file(table, backup_date_str, csv_data_file)
           File.delete(csv_data_file)
+
+        elsif tsv_data_file.exist?
+          print " - Importing #{File.basename(tsv_data_file)} ... "
+          tsv_data_file = import_tsv_file(table, backup_date_str, tsv_data_file)
+          File.delete(tsv_data_file)
         end
 
         if index_file.exist?
@@ -95,7 +107,7 @@ module MysqlTruck
           execute_sql_file(table, backup_date_str, index_file)
           index_file.delete
         end
-        
+
       end
 
       puts "Backup loaded."
@@ -138,6 +150,23 @@ module MysqlTruck
       file_path
     end
 
+    def import_tsv_file(table, backup_date_str, file_path)
+      if config[:date_suffix]
+        old_file_path = file_path
+        file_path = file_path.to_s.gsub("/#{table}.data.", "/#{table}_#{backup_date_str.gsub(/-/, "")}.")
+        # move file
+        `mv #{old_file_path} #{file_path}`
+      end
+
+      time = benchmark do
+        `mysqlimport --local --compress #{db_connection_options} #{file_path}`
+      end
+
+      print "complete (#{formatted_time time}).\n"
+
+      file_path
+    end
+
     def download_file(key)
       filename = File.basename(key.name)
 

data/lib/mysql_truck/version.rb

@@ -1,3 +1,3 @@
 module MysqlTruck
-  VERSION = "0.5.7.1"
+  VERSION = "0.6.0"
 end

data/lib/mysql_truck.rb

@@ -1,4 +1,6 @@
 require "right_aws"
+require "mysql"
+require "csv"
 require "fileutils"
 require "pathname"
 

data/mysql_truck.gemspec

@@ -16,5 +16,6 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency "right_aws"
+  s.add_runtime_dependency "right_aws", "~> 2.1.0"
+  s.add_runtime_dependency "mysql", "~> 2.8.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mysql_truck
 version: !ruby/object:Gem::Version
-  version: 0.5.7.1
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -11,24 +11,40 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-15 00:00:00.000000000 Z
+date: 2013-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: right_aws
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: mysql
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.8.1
 description: Mysql database backup tool. Dumps/Loads to/from S3.
 email:
 - peter@paydrotalks.com