brbackup 0.1.0

data/MIT-LICENSE

@@ -0,0 +1,18 @@
+Copyright (c) 2009 Tung Nguyen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of 
+this software and associated documentation files (the "Software"), to deal in 
+the Software without restriction, including without limitation the rights to 
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 
+the Software, and to permit persons to whom the Software is furnished to do so, 
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all 
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,20 @@
+Brbackup
+=======
+
+Summary
+
+Install
+-------
+
+<pre>
+sudo gem install brbackup --source http://gemcutter.org
+</pre>
+
+
+Compatibility
+-------------
+
+Tested with Ruby 1.8.6.
+
+Thanks
+------

data/Rakefile

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'rake'
+require 'rake/gempackagetask'
+require 'spec/rake/spectask'
+require 'gemspec'
+
+desc "Generate gemspec"
+task :gemspec do
+  File.open("#{Dir.pwd}/#{GEM_NAME}.gemspec", 'w') do |f|
+    f.write(GEM_SPEC.to_ruby)
+  end
+end
+
+desc "Install gem"
+task :install do
+  Rake::Task['gem'].invoke
+  `gem install pkg/#{GEM_NAME}*.gem`
+  `rm -Rf pkg`
+end
+
+desc "Package gem"
+Rake::GemPackageTask.new(GEM_SPEC) do |pkg|
+  pkg.gem_spec = GEM_SPEC
+end
+
+desc "Run specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ["--format", "specdoc", "--colour"]
+  t.spec_files = FileList["spec/**/*_spec.rb"]
+end
+
+task :default => :spec do
+end
+
+# You can delete this after you use it
+desc "Rename project"
+task :rename do
+  name = ENV['NAME'] || File.basename(Dir.pwd)
+  class_name = name.split('_').collect{|x| x.capitalize}.join("")
+  begin
+    dir = Dir['**/gem_template*']
+    from = dir.pop
+    if from
+      rb = from.include?('.rb')
+      to = File.dirname(from) + "/#{name}#{'.rb' if rb}"
+      FileUtils.mv(from, to)
+    end
+  end while dir.length > 0
+  Dir["**/*"].each do |path|
+    next if path.include?('Rakefile')
+    if File.file?(path)
+      `sed -i "" 's/gem_template/#{name}/g' #{path}`
+      `sed -i "" 's/GemTemplate/#{class_name}/g' #{path}`
+    end
+  end
+end

data/bin/brbackup

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/brbackup'
+
+BR::Backups.run(ARGV)

data/gemspec.rb

@@ -0,0 +1,19 @@
+GEM_NAME = 'brbackup'
+GEM_FILES = FileList['**/*'] - FileList['coverage', 'coverage/**/*', 'pkg', 'pkg/**/*']
+GEM_SPEC = Gem::Specification.new do |s|
+  # == CONFIGURE ==
+  s.author = "Tung Nguyen"
+  s.email = "tongueroo@gmail.com"
+  s.homepage = "http://github.com/tongueroo/#{GEM_NAME}"
+  s.summary = "brbackup summary"
+  # == CONFIGURE ==
+  s.executables = ["brbackup"]
+  s.add_dependency('builder', '>=2.1.2')
+  s.extra_rdoc_files = [ "README.markdown" ]
+  s.files = GEM_FILES.to_a
+  s.has_rdoc = false
+  s.name = GEM_NAME
+  s.platform = Gem::Platform::RUBY
+  s.require_path = "lib"
+  s.version = "0.1.0"
+end

data/lib/brbackup.rb

@@ -0,0 +1,342 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'aws/s3'
+require 'yaml'
+require 'pp'
+require 'optparse'
+require 'open4'
+require 'fileutils'
+require 'logger'
+
+module AWS::S3
+  class S3Object
+    def <=>(other)
+      DateTime.parse(self.about['last-modified']) <=> DateTime.parse(other.about['last-modified'])
+    end
+  end
+end
+
+# Usage:
+# Simple:
+# restores bleacherreport_production from production to staging db
+#   $ brbackup -f prod_br --clone bleacherreport
+#
+# More examples: 
+# Specifying more options
+#   $ brbackup -f prod_ss --clone cmservice --logger /tmp/brbackup.log -c .mysql.backups.yml 
+#   $ brbackup -f prod_ss --clone cmservice --logger /tmp/brbackup.log -c .mysql.backups.yml 
+#
+# Notes:
+# Format:
+# <bucket_name>/<env>.<db>/<filename>
+# Example : 
+# ey-backup-fe5a050b997d.s3.amazonaws.com/...
+#   prod_br.bleacherreport_production/bleacherreport_production.2010-03-07T09-10-01.sql.gz
+# 
+# The --clone option only needs the prefix of the database name because the amazon searches based
+# on the prefix of <env>.<db>
+# The filename of the dumpfile is then taken and then gsubed _production -> _staging
+# 
+module BR
+  class DatabaseEngine
+    def self.register_as(name)
+      BR::Backups::ENGINES[name] = self
+    end
+    
+    attr_accessor :logger
+
+    def initialize(backups)
+      @backups = backups
+    end
+
+    def dump_database(name)
+      raise "Implement #dump_database in #{self.class}"
+    end
+
+    def dbuser
+      @backups.config[:dbuser]
+    end
+
+    def dbpass
+      @backups.config[:dbpass]
+    end
+    
+    def log(msg)
+      @logger.info(msg)
+    end
+  end
+
+  class Backups
+    ENGINES = {}
+    
+    def self.run(args)
+      options = {}
+      
+      # Build a parser for the command line arguments
+      opts = OptionParser.new do |opts|
+        opts.version = VERSION
+
+        opts.banner = "Usage: brbackup [-flag] [argument]"
+        opts.define_head "brbackup: clone db backups across environments"
+        opts.separator '*'*80
+        
+        opts.on("-f", "--from ENVIRONMENT", "EY Cloud environment name want to clone from : prod_br, beta, alpha, etc") do |env|
+          options[:env] = env
+        end
+        
+        opts.on("-l", "--list-backup DATABASE", "List mysql backups for DATABASE") do |db|
+          options[:db] = (db || 'all')
+          options[:command] = :list
+        end
+        
+        # opts.on("-n", "--names DB1,DB2,DB3", "Only restore these databases") do |n|
+        #   options[:databases] = n.split(',')
+        # end
+        
+        opts.on("-c", "--config CONFIG", "Use config file.") do |config|
+          options[:config] = config
+        end
+        
+        opts.on("-d", "--download BACKUP_INDEX", "download the backup specified by index. Run brbackup -l to get the index.") do |index|
+          options[:command] = :download
+          options[:index] = index
+        end
+        
+        opts.on("--clone DB_NAME", "Clones production database to staging") do |db_name|
+          options[:command] = :clone
+          options[:db_name] = db_name
+        end
+
+        opts.on("--logger /path/to/logger", "Path to log path, default is stdout") do |logger|
+          options[:logger_path] = logger
+        end
+
+        # opts.on("-r", "--restore BACKUP_INDEX", "Download and apply the backup specified by index WARNING! will overwrite the current db with the backup. Run brbackup -l to get the index.") do |index|
+        #   options[:command] = :restore
+        #   options[:index] = index
+        # end
+
+      end
+
+      opts.parse!(args)
+
+      options[:engine] ||= 'mysql'
+      options[:config] ||= "/etc/.#{options[:engine]}.backups.yml"
+
+      brb = new(options)
+
+      case options[:command]
+      when :list
+        brb.list options[:db], true
+      when :download
+        brb.download(options[:index])
+      # when :restore
+      #   brb.restore(options[:index])
+      when :clone
+        brb.clone(options[:db_name])
+      end
+    rescue SystemExit
+      exit 1
+    rescue Exception => e
+      $stderr.puts "An unknown exception was raised"
+      $stderr.puts e.inspect
+      $stderr.puts e.backtrace
+      $stderr.puts
+      raise
+    end
+    
+    def initialize(options)
+      if options[:logger_path]
+        @logger = Logger.new(options[:logger_path])
+      else
+        @logger = Logger.new($stdout)
+      end
+
+      engine_klass = ENGINES[options[:engine]] || raise("Invalid database engine: #{options[:engine].inspect}")
+      @engine = engine_klass.new(self)
+      @engine.logger = @logger
+
+      load_config(options[:config])
+
+      AWS::S3::Base.establish_connection!(
+          :access_key_id     => config[:aws_secret_id],
+          :secret_access_key => config[:aws_secret_key]
+        )
+      @databases = options[:databases] || config[:databases]
+      @keep = config[:keep]
+      @bucket = "ey-backup-#{Digest::SHA1.hexdigest(config[:aws_secret_id])[0..11]}"
+      @tmpname = "#{Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')}.sql.gz"
+      @env = options[:env] || config[:env]
+      FileUtils.mkdir_p '/mnt/backups'
+      FileUtils.mkdir_p '/mnt/tmp'
+      begin
+        AWS::S3::Bucket.find(@bucket)
+      rescue AWS::S3::NoSuchBucket
+        AWS::S3::Bucket.create(@bucket)
+      end
+      
+      FileUtils.mkdir_p self.backup_dir
+    end
+    attr_reader :config
+    
+    def load_config(filename)
+      if File.exist?(filename)
+        @config = YAML::load(File.read(filename))
+      else
+        log "You need to have a backup file at #{filename}"
+        $stderr.puts "You need to have a backup file at #{filename}"
+        exit 1
+      end
+    end
+    
+    def new_backup
+      @databases.each do |db|
+        backup_database(db)
+      end  
+    end
+    
+    def backup_database(database)
+      File.open("#{self.backup_dir}/#{database}.#{@tmpname}", "w") do |f|
+        log "doing database: #{database}"
+        @engine.dump_database(database, f)
+      end
+
+      File.open("#{self.backup_dir}/#{database}.#{@tmpname}") do |f|
+        path = "#{@env}.#{database}/#{database}.#{@tmpname}"
+        AWS::S3::S3Object.store(path, f, @bucket, :access => :private)
+        log "successful backup: #{database}.#{@tmpname}"
+      end
+    end
+
+    def download(index)
+      idx, db = index.split(":")
+      raise Error, "You didn't specify a database name: e.g. 1:rails_production" unless db
+
+      if obj = list(db)[idx.to_i]
+        filename = normalize_name(obj)
+        log "downloading: #{filename}"
+        File.open(filename, 'wb') do |f|
+          print "."
+          obj.value {|chunk| f.write chunk }
+        end
+        log ""
+        log "finished"
+        [db, filename]
+      else
+        raise BackupNotFound, "No backup found for database #{db.inspect}: requested index: #{idx}"
+      end
+    end
+    
+    def restore(index)
+      db, filename = download(index)
+      File.open(filename) do |f|
+        @engine.restore_database(db, f)
+      end
+    end
+    
+    def clone(db_name)
+      index = most_recent_index(db_name)
+      db, filename = download(index)
+      log "db #{db.inspect}"
+      log "filename #{filename.inspect}"
+      staging_name = filename.split('.')[0].gsub('_production', '_staging')
+      File.open(filename) do |f|
+        @engine.clone_database(staging_name, f)
+      end
+    end
+    
+    def cleanup
+      begin
+        list('all',false)[0...-(@keep*@databases.size)].each do |o| 
+          log "deleting: #{o.key}"
+          o.delete
+        end
+      rescue AWS::S3::S3Exception, AWS::S3::Error
+        nil # see bucket_minder cleanup note regarding S3 consistency
+      end
+    end
+    
+    def normalize_name(obj)
+      obj.key.gsub(/^.*?\//, '')
+    end
+    
+    def find_obj(name)
+      AWS::S3::S3Object.find name, @bucket
+    end
+    
+    def list(database='all', printer = false)
+      puts "Listing database backups for #{database}" if printer
+      backups = []
+      if database == 'all'
+        @databases.each do |db|
+          backups << AWS::S3::Bucket.objects(@bucket, :prefix => "#{@env}.#{db}")
+        end
+        backups = backups.flatten.sort
+      else
+        backups = AWS::S3::Bucket.objects(@bucket, :prefix => "#{@env}.#{database}").sort
+      end
+      if printer
+        puts "#{backups.size} backup(s) found"
+        backups.each_with_index do |b,i|
+          puts "#{i}:#{database} #{normalize_name(b)}"
+        end
+      end    
+      backups
+    end
+    
+    # dirty way to get most recent index
+    def most_recent_index(db)
+      index = list(db).size - 1
+      "#{index}:#{db}"
+    end
+    
+    protected
+    def backup_dir
+      "/mnt/tmp"
+    end
+    
+    def log(msg)
+      @logger.info(msg)
+    end
+  end
+  
+  class MysqlDatabase < DatabaseEngine
+    register_as 'mysql'
+
+    def dump_database(name, io)
+      single_transaction = db_has_myisam?(name) ? '' : '--single-transaction'
+      Open4.spawn ["mysqldump -u#{dbuser} #{password_option} #{single_transaction} #{name} | gzip -c"], :stdout => io
+    end
+
+    def db_has_myisam?(name)
+      query = "SELECT 1 FROM information_schema.tables WHERE table_schema='#{name}' AND engine='MyISAM' LIMIT 1;"
+      %x{mysql -u #{dbuser} #{password_option} -N -e"#{query}"}.strip == '1'
+    end
+
+    def restore_database(name, io)
+      log "mock restoring database..."
+      # Open4.spawn ["gzip -dc | mysql -u#{dbuser} #{password_option} #{name}"], :stdin => io
+    end
+    
+    def clone_database(staging_name, io)
+      log "dropping #{staging_name} database"
+      cmd = "mysql -u#{dbuser} #{password_option} -e 'drop database #{staging_name}'"
+      Open4.popen4 cmd do |pid, stdin, stdout, stderr|
+        log stdout.read
+      end
+      log "creating #{staging_name} database"
+      Open4.popen4 "mysql -u#{dbuser} #{password_option} -e 'create database #{staging_name}'" do |pid, stdin, stdout, stderr|
+        log stdout.read
+      end
+      log "loading new dump..."
+      Open4.spawn ["gzip -dc | mysql -u#{dbuser} #{password_option} #{staging_name}"], :stdin => io
+      log "new dump loaded."
+    end
+
+    def password_option
+      dbpass.nil? || dbpass.blank? ? "" : "-p'#{dbpass}'"
+    end
+    
+  end
+end

data/spec/brbackup/brbackup.rb

@@ -0,0 +1,6 @@
+require File.expand_path("#{File.dirname(__FILE__)}/../spec_helper")
+
+describe Brbackup do
+
+
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,18 @@
+$testing = true
+SPEC = File.dirname(__FILE__)
+$:.unshift File.expand_path("#{SPEC}/../lib")
+
+require 'brbackup'
+require 'pp'
+require 'rubygems'
+
+Spec::Runner.configure do |config|
+end
+
+# For use with rspec textmate bundle
+def debug(object)
+  puts "<pre>"
+  puts object.pretty_inspect.gsub('<', '&lt;').gsub('>', '&gt;')
+  puts "</pre>"
+end
+

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification 
+name: brbackup
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Tung Nguyen
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-03-21 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: builder
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 1
+        - 2
+        version: 2.1.2
+  type: :runtime
+  version_requirements: *id001
+description: 
+email: tongueroo@gmail.com
+executables: 
+- brbackup
+extensions: []
+
+extra_rdoc_files: 
+- README.markdown
+files: 
+- bin/brbackup
+- gemspec.rb
+- lib/brbackup.rb
+- MIT-LICENSE
+- Rakefile
+- README.markdown
+- spec/brbackup/brbackup.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/tongueroo/brbackup
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: brbackup summary
+test_files: []
+