mysql_truck 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in mysql_truck.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/mysql_truck

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'mysql_truck'
+require 'yaml'
+
+options = {}
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename(__FILE__)} (dump|load) [options]"
+
+  opts.separator ""
+  opts.separator "Required Options:"
+
+  opts.on("-c", "--config FILE", "Configuration yaml file") do |file|
+    options[:config] = YAML.load_file(Pathname.new(file).expand_path)
+  end
+
+  opts.separator ""
+  opts.separator "Or specify options individually:"
+  opts.separator "(options here override options from the config if specified)"
+
+  opts.on("-H", "--host HOST", "Database host") do |host|
+    options[:host] = host
+  end
+
+  opts.on("-u", "--user USERNAME", "Database user") do |user|
+    options[:username] = user
+  end
+
+  opts.on("-p", "--password PASSWORD", "Database password") do |password|
+    options[:password] = password
+  end
+
+  opts.on("-d", "--database DATABASE_NAME", "Database name") do |db|
+    options[:database] = db
+  end
+
+  opts.on("-a", "--access-key KEY", "S3 Access Key") do |key|
+    options[:s3_access_key] = key
+  end
+
+  opts.on("-s", "--secret-access-key KEY", "S3 Secret Acess Key") do |key|
+    options[:s3_secret_access_key] = key
+  end
+
+  opts.on("-t", "--skip-tables TABLES",
+          "List of tables to skip separated by commas.") do |tables|
+    options[:skip_tables] = tables.split(",")
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end
+
+begin
+  parser.parse!(ARGV)
+
+  if options[:config]
+    config = {}
+    options.delete(:config).each do |k,v|
+      config[k.intern] = v
+    end
+
+    options = config.merge(options)
+  end
+
+  missing_opts = []
+  [ :s3_access_key, :s3_secret_access_key,
+    :host, :username, :database
+  ].each do |opt|
+    missing_opts << opt unless options.include?(opt)
+  end
+
+  if missing_opts.size > 0
+    puts "Missing the following options:"
+    missing_opts.each { |o| puts " * #{o}" }
+    puts
+    puts parser
+    exit
+  end
+rescue OptionParser::ParseError => e
+  puts "\nPlease ensure you have the correct options\n\n"
+  puts parser
+  exit
+end
+
+MysqlTruck.run(ARGV.first, options)

data/lib/mysql_truck.rb

@@ -0,0 +1,240 @@
+require "mysql_truck/version"
+require "right_aws"
+require 'fileutils'
+require 'pathname'
+
+# MysqlTruck
+#
+# MySQL backup tool that stores backups in S3.
+#
+# Requires server to have right_aws installed
+module MysqlTruck
+  include FileUtils
+
+  def self.run(action, config)
+    case action.intern
+    when :dump
+      Dumper.new(config).dump
+    when :load
+      Loader.new(config).load_latest
+    else
+      puts "Unknown action #{action}"
+    end
+  end
+
+  module Helper
+    include FileUtils
+
+    def config
+      @config
+    end
+
+    def initialize_s3
+      @s3 = RightAws::S3.new(
+        config[:s3_access_key],
+        config[:s3_secret_access_key])
+      @bucket = @s3.bucket("8tr.db_backups")
+    end
+
+    def db_connection_options
+      opts = %Q[ -u #{config[:username]} ]
+      opts += %Q[ -p"#{config[:password]}" ] unless config[:password].nil?
+      opts += %Q[ -h #{config[:host]} --default-character-set=utf8 ]
+      opts += %Q[ #{config[:database]} ]
+      opts
+    end
+
+    def csv_options
+      " --fields-enclosed-by=\\\" --fields-terminated-by=, "
+    end
+
+    def tmp_path
+      unless @tmp_path
+        @tmp_path = config[:tmp_dir] || Pathname.new("/data/s3backup")
+        @tmp_path = @tmp_path.join(@time.to_i.to_s) if @time # Only set with the Dumper class
+      end
+
+      @tmp_path
+    end
+
+  end
+
+  class Dumper
+    include FileUtils
+    include Helper
+
+    def initialize(config)
+      @config = config
+      @time = Time.now
+
+      initialize_s3
+      initialize_directories
+    end
+
+    def dump
+      dump_data
+      upload
+
+    ensure
+      # rm_r tmp_path, :force => true
+    end
+
+    def dump_data
+      tables.each do |table|
+        schema_file = tmp_path.join("#{table}.sql")
+        csv_file = tmp_path.join("#{table}.txt")
+        puts "Dumping #{table}."
+
+        # This command creates a table_name.sql and a table_name.txt file
+        cmd = "mysqldump --quick -T #{tmp_path} "
+        cmd += csv_options
+        cmd += "#{db_connection_options} #{table}"
+        puts cmd
+        `#{cmd}`
+
+        path, file = csv_file.split
+        csv_file = path.join("#{file.basename(".txt")}.csv")
+        mv path.join(file), csv_file
+
+        puts "gziping #{schema_file}."
+        `gzip #{schema_file}`
+
+        puts "gziping #{csv_file}."
+        `gzip #{csv_file}`
+
+        puts "#{table} dumped.\n\n"
+      end
+    end
+
+    def upload
+      Dir["#{tmp_path}/*"].each do |file|
+        upload_file file
+      end
+      puts "Finished uploading backups."
+    end
+
+    private
+
+    def initialize_directories
+      mkdir_p tmp_path
+    end
+
+    def upload_file(local_file)
+      path = Pathname.new(local_file)
+      s3_path = bucket_path.join(path.basename)
+      @bucket.put(s3_path, open(path), {}, nil, {
+        'x-amz-storage-class' => 'REDUCED_REDUNDANCY'
+      })
+    end
+
+    def tables
+      unless @tables
+        res = `mysql #{db_connection_options} -e "SHOW TABLES"`
+        # @tables = res.split[1..-1]
+        @tables = res.split[1..10]
+      end
+      @tables
+    end
+
+    def bucket_path
+      @bucket_path ||= Pathname.new("mysql").join(@time.strftime("%Y-%m-%d-%H-%M"))
+    end
+  end # class Dumper
+
+
+  class Loader
+    include Helper
+    include FileUtils
+
+    def initialize(config)
+      @config = config
+      initialize_s3
+    end
+
+    def skip_tables
+      config[:skip_tables] || []
+    end
+
+    def load_latest
+      prefix = backups.first
+
+      tmp_dir = tmp_path.join(prefix.split("/").last)
+      mkdir_p tmp_dir # Creates a directory for day of the backup downloaded
+
+      puts "Downloading backups ..."
+      @bucket.keys(:prefix => prefix).each do |key|
+        filename = File.basename(key.name)
+
+        next if filename.match(/\.csv\.gz$/) && skip_tables.include?(File.basename(filename, ".csv.gz"))
+
+        print " - Downloading #{filename} ... "
+        File.open(tmp_dir.join(filename), "wb") do |f|
+          f.write key.get
+        end
+        print "complete.\n"
+
+        # gunzip file
+        print " -- Inflating #{filename} ... "
+        `gunzip #{tmp_dir.join(filename)}`
+        print "complete.\n"
+      end
+
+      # Load schemas
+      puts "Loading schema."
+      cmd = "cat #{tmp_dir}/*.sql | "
+      cmd += "mysql #{db_connection_options}"
+      puts cmd
+      `#{cmd}`
+
+      # Load data
+
+      puts "Loading data."
+      cmd = "mysqlimport #{db_connection_options}"
+      cmd += csv_options
+      Dir["#{tmp_dir}/*.csv"].each do |table|
+        print " - Importing #{File.basename(table, ".csv")} ... "
+        puts cmd + " " + table
+        `#{cmd} #{table}`
+        print "complete.\n"
+      end
+
+      puts "Backup loaded."
+    ensure
+      rm_r tmp_dir, :force => true
+    end
+
+    # Get a list of backups stored on S3.
+    #
+    # Returns an array of s3 paths that look like:
+    #
+    #   mysql/YYYY-MM-DD-HH-MM
+    #
+    # Array elements are sorted with the latest date first.
+    def backups
+      unless @backups
+        @backups = []
+        # Backups are stored in the mysql/ directory
+        @bucket.s3.interface.incrementally_list_bucket(@bucket.name, {
+          :prefix => "mysql/", :delimiter => "/"
+        }) do |item|
+          @backups += item[:common_prefixes]
+        end
+        @backups = @backups.sort { |a,b| b <=> a }
+      end
+      @backups
+    end
+
+    def tables
+      unless @tables
+        res = `mysql #{db_connection_options} -e "SHOW TABLES"`
+        # @tables = res.split[1..-1]
+        @tables = res.split[1..10]
+      end
+      @tables
+    end
+
+    def initialize_directories
+      mkdir_p tmp_dir
+    end
+  end # class Loader
+end

data/lib/mysql_truck/version.rb

@@ -0,0 +1,3 @@
+module MysqlTruck
+  VERSION = "0.0.1"
+end

data/mysql_truck.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "mysql_truck/version"
+
+Gem::Specification.new do |s|
+  s.name        = "mysql_truck"
+  s.version     = MysqlTruck::VERSION
+  s.authors     = ["Peter Bui", "8tracks"]
+  s.email       = ["peter@paydrotalks.com"]
+  s.homepage    = ""
+  s.summary     = %q{Mysql database backup tool. Dumps/Loads to/from S3.}
+  s.description = %q{Mysql database backup tool. Dumps/Loads to/from S3.}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "right_aws"
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: mysql_truck
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Peter Bui
+- 8tracks
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-09-15 00:00:00.000000000 -07:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: right_aws
+  requirement: &2163188460 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2163188460
+description: Mysql database backup tool. Dumps/Loads to/from S3.
+email:
+- peter@paydrotalks.com
+executables:
+- mysql_truck
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Rakefile
+- bin/mysql_truck
+- lib/mysql_truck.rb
+- lib/mysql_truck/version.rb
+- mysql_truck.gemspec
+has_rdoc: true
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Mysql database backup tool. Dumps/Loads to/from S3.
+test_files: []