s3mybackup 0.0.2

data/README.rdoc

@@ -0,0 +1,6 @@
+= s3backup
+
+Describe your project here
+
+:include:s3mybackup.rdoc
+

data/bin/s3mybackup

@@ -0,0 +1,231 @@
+#!/usr/bin/env ruby
+require 'gli'
+require "rubygems"
+require "aws-sdk"
+require "yaml"
+require "fileutils"
+require "socket"
+require File.join(File.dirname(__FILE__), "../lib", 's3mybackup')
+
+include GLI::App
+
+program_desc 'Tool for backing up and restoring mysql databases to and from S3.'
+
+version = S3mybackup::VERSION
+
+config_file = File.join(".","config.yml")
+
+unless File.exist?(config_file) then
+  raise "File #{config_file} does not exist."
+end
+
+defaults = YAML.load(File.read(config_file))
+
+unless defaults.kind_of?(Hash) then
+  raise "config.yml is formatted incorrectly.  Please use the following format: \naccess_key_id: YOUR_ACCESS_KEY_ID\nsecret_access_key: YOUR_SECRET_ACCESS_KEY"
+end
+
+desc 'User'
+default_value 'backup'
+arg_name 'Backup user'
+flag [:u, :user]
+
+desc 'Password'
+arg_name 'Password'
+flag [:p, :password]
+
+desc 'Directory where mysql writes binary logs. log-bin = in my.cnf'
+default_value defaults[:mysql_bin_log_dir]
+arg_name 'mysql_bin_log_dir'
+flag [:l, :mysql_bin_log_dir]
+
+desc 'Temporary directory where files are stored before writing to and after download from S3'
+default_value defaults[:temp_dir]
+arg_name 'mysql_bin_log_dir'
+flag [:t, :temp_dir]
+
+desc 'Access key S3'
+default_value defaults[:access_key_id]
+arg_name 'access_key_id'
+flag [:a, :access_key_id]
+
+desc 'Secret access key S3'
+default_value defaults[:secret_access_key]
+arg_name 'secret_access_key'
+flag [:s, :secret_access_key]
+
+desc 'Secret access key S3'
+default_value defaults[:s3_endpoint]
+arg_name 's3_endpoint'
+flag [:e, :s3_endpoint]
+
+desc 'Bucket for backups'
+arg_name 'Bucket'
+flag [:b, :bucket]
+
+desc 'Creates a full backup to S3'
+arg_name 'Describe arguments to full here'
+
+command :full do |c|
+
+  c.action do |global_options, options, args|
+
+    bucket = get_bucket(global_options)
+
+    database = args[0]
+
+    database_dir = get_database_dir(database)
+
+    dump_file_name = get_full_backup_file_name
+
+    # delete all incremental backup files
+    bucket.objects.with_prefix(database_dir).delete_if { |o| o.key.include?("-bin.") }
+
+    user = global_options[:user]
+    password = global_options[:password]
+
+
+    # assumes the bucket's empty
+    dump_file = "#{@temp_dir}/#{dump_file_name}"
+
+    cmd = "mysqldump --quick --single-transaction --create-options -u #{user} --flush-logs --master-data=2 --delete-master-logs"
+    cmd += " -p'#{password}'" unless password.nil?
+    cmd += " #{database} | gzip > #{dump_file}"
+    run_system(cmd)
+
+    bucket.objects.create("#{database_dir}/#{dump_file_name}", open(dump_file))
+  end
+end
+
+desc 'Executes incremental backup to S3'
+arg_name 'database to backup'
+command :inc do |c|
+  c.action do |global_options, options, args|
+    user = global_options[:user]
+    password = global_options[:password]
+    execute_sql "flush logs", user, password
+    logs = Dir.glob("#{@mysql_bin_log_dir}/*-bin.[0-9]*").sort
+    logs_to_archive = logs[0..-2] # all logs except the last
+
+    bucket = get_bucket global_options
+
+    logs_to_archive.each do |log|
+      bucket.objects.create("#{get_database_dir(args[0])}/#{File.basename(log)}", open(log))
+    end
+    if logs[-1] then
+      execute_sql "purge master logs to '#{File.basename(logs[-1])}'", user, password
+    end
+  end
+end
+
+desc 'Describe restore here'
+arg_name 'Describe arguments to restore here'
+command :restore do |c|
+  c.desc 'Restore the binary logs'
+  c.switch [:l, :logs]
+
+  c.desc 'IP address of server without the dots'
+  c.arg_name 'Ip'
+  c.default_value get_ip
+  c.flag [:i, :ip]
+
+  c.desc 'Time of the backup to restore'
+  c.arg_name 'Time'
+  c.flag [:t, :time]
+  c.action do |global_options, options, args|
+    time = options[:time]
+
+    unless time then
+      raise "Option time is required"
+    end
+
+    user = global_options[:user]
+    password = global_options[:password]
+
+    ip = options[:ip]
+
+    bucket = get_bucket global_options
+
+    from_database = args[0]
+    to_database = args[1]
+
+    unless to_database then
+      to_database = from_database
+    end
+
+    database_dir = get_database_dir(from_database, ip)
+
+    full_backup_file_name = get_full_backup_file_name(time)
+
+    backup_file_key = "#{database_dir}/#{full_backup_file_name}"
+    unless bucket.objects[backup_file_key].exists? then
+      raise "Dump #{backup_file_key} does not not exist in bucket #{bucket.name}"
+    end
+    retrieve_file(bucket, database_dir, full_backup_file_name)
+
+    # restore the dump file
+    cmd = "gunzip -c #{@temp_dir}/#{full_backup_file_name} | mysql -u #{user} "
+    cmd += " -p'#{password}' " unless password.nil?
+    cmd += " #{to_database}"
+    run_system cmd
+
+    if options[:logs] then
+
+      binary_log_objects = bucket.objects.with_prefix(database_dir).select { |obj| obj.key.match(/.*-bin\.[0-9]{6}/) }
+      binary_log_objects.each do |o|
+        File.open("#{@temp_dir}/#{File.basename(o.key)}", 'w') do |f|
+          o.read do |chunk|
+            f.write(chunk)
+          end
+        end
+      end
+
+
+      logs = Dir.glob("#{@temp_dir}/*-bin.[0-9]*").sort
+
+      # restore the binary log files
+      logs.each do |log|
+        # The following will be executed for each binary log file
+        cmd = "mysqlbinlog --database=#{from_database} #{log} | mysql -u #{user} "
+        cmd += " -p'#{password}' " unless password.nil?
+        cmd += " #{to_database}"
+        run_system cmd
+      end
+    end
+  end
+end
+
+pre do |global, command, options, args|
+  if args.empty? then
+    raise "Database name is required."
+  end
+
+
+
+
+
+  @mysql_bin_log_dir = global[:mysql_bin_log_dir]
+  @temp_dir = global[:temp_dir]
+
+  AWS.config({
+                 :access_key_id => global[:access_key_id],
+                 :secret_access_key => global[:secret_access_key],
+                 :s3_endpoint => global[:s3_endpoint]
+             })
+
+  FileUtils.mkdir_p @temp_dir
+
+  true
+end
+
+post do |global, command, options, args|
+  FileUtils.rm_rf(@temp_dir)
+end
+
+on_error do |exception|
+  # Error logic here
+  # return false to skip default error handling
+  true
+end
+
+exit run(ARGV)

data/lib/s3mybackup/version.rb

@@ -0,0 +1,3 @@
+module S3mybackup
+  VERSION = '0.0.2'
+end

data/lib/s3mybackup.rb

@@ -0,0 +1,47 @@
+require 's3mybackup/version.rb'
+
+def run_system(command)
+  result = system(command)
+  raise("error, process exited with status #{$?.exitstatus}") unless result
+end
+
+def execute_sql(sql,user,password)
+  cmd = "mysql -u #{user} -e \"#{sql}\""
+  cmd += " -p'#{password}' " unless password.nil?
+  run_system cmd
+end
+
+def get_bucket(global_options)
+  s3 = AWS::S3.new
+
+  bucket_name = global_options[:bucket]
+  bucket = s3.buckets[bucket_name]
+
+  if not bucket.exists? then
+    bucket = s3.buckets.create(bucket_name)
+  end
+  bucket
+end
+
+def get_ip
+  IPSocket.getaddress(Socket.gethostname).gsub!(/\./, "")
+end
+
+def get_database_dir(database,ip = get_ip)
+  database_dir = "#{ip}/#{database}"
+end
+
+def get_full_backup_file_name(time_string = (Time.now).strftime('%Y%m%d%H%M%S'))
+  "#{time_string}dump.sql.gz"
+end
+
+def retrieve_file(bucket,database_dir,file_name)
+
+  full_dump = bucket.objects["#{database_dir}/#{file_name}"]
+
+  File.open("#{@temp_dir}/#{file_name}", 'w') do |f|
+    full_dump.read do |chunk|
+      f.write(chunk)
+    end
+  end
+end

data/s3mybackup.rdoc

@@ -0,0 +1,5 @@
+= s3backup
+
+Generate this with
+    s3backup rdoc
+After you have described your command line interface

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification 
+name: s3mybackup
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
+platform: ruby
+authors: 
+- Joris Wijlens
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-12-30 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rdoc
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: aruba
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: gli
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 2
+        - 5
+        - 3
+        version: 2.5.3
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: aws-sdk
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 1
+        - 7
+        - 1
+        version: 1.7.1
+  type: :runtime
+  version_requirements: *id005
+description: "    Options can be configured in a file config.yml located in the directory from where command is executed.\n"
+email: joris@smartworkx.com
+executables: 
+- s3mybackup
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- s3mybackup.rdoc
+files: 
+- bin/s3mybackup
+- lib/s3mybackup/version.rb
+- lib/s3mybackup.rb
+- README.rdoc
+- s3mybackup.rdoc
+has_rdoc: true
+homepage: http://www.smartworkx.nl
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --title
+- s3mybackup
+- --main
+- README.rdoc
+- -ri
+require_paths: 
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.4.2
+signing_key: 
+specification_version: 3
+summary: Command suite for backing up MySQL databases to S3
+test_files: []
+