backup_utility 1.0.0

data/lib/backup_utility.rb

@@ -0,0 +1,3 @@
+module BackupUtility; end
+require 'backup_utility/postgres'
+require 'backup_utility/mongo'

data/lib/backup_utility/base.rb

@@ -0,0 +1,228 @@
+require 'fileutils'
+
+CODE_DIR = '/home/twist/code'
+BACKUP_DIR = '/home/twist/backups'
+SHARED_DIR = '/mnt/shared_storage2/database/backups'
+class BackupUtility::Base
+  def initialize(backup_info)
+    @backup_dir = backup_info[:backup_dir] || BACKUP_DIR
+    @shared_dir = backup_info[:shared_dir] || SHARED_DIR
+    @code_dir = backup_info[:code_dir] || CODE_DIR
+    @backup_machine = backup_info[:backup_machine] || 'a1-stats'
+
+    @send_to_shared = backup_info.fetch(:send_to_shared, true)
+    @send_to_s3 = backup_info.fetch(:send_to_s3, true)
+    @move_backups = backup_info.fetch(:move_backups, true)
+    @use_code = backup_info.fetch(:use_code, false)
+    @use_scp = backup_info.fetch(:use_scp, false)
+    @save = backup_info.fetch(:save, true)
+    @clean_shared_age = backup_info[:clean_shared_age]
+    @clean_shared_age = ::Rails.env == 'production' ? 30 : 14 if @clean_shared_age.blank?
+
+    @clean_backups_age = backup_info[:clean_backups_age]
+    if @clean_backups_age.blank?
+      if ::Rails.env == 'production'
+        @cleanup_backups_age = @move_backups ? 7 : 14
+      else
+        @cleanup_backups_age = @move_backups ? 3 : 7
+      end
+    end
+    @last_dst = nil
+  end
+
+  def log_and_time(out)
+    log(out)
+    start = Time.now
+    ret = yield
+    took = Time.now - start
+    log("took #{took}")
+    ret
+  end
+ 
+  def log(out)
+    puts "#{Time.now} - #{out}"
+  end
+
+  def backup(settings = {}, append = nil)
+    now = Time.now
+    storage_name, working_dir, backup_file = get_paths(settings, append)
+    if !perform_dump(settings, working_dir)
+      log("could not perform dump properly")
+      return false
+    end
+    if custom?
+      # no need to tar up the directory, all we need to do is rename
+      # the file in the backup_dir to the backup name
+      log("moving file #{@last_dst} to #{backup_file}")
+      File.mv(@last_dst, backup_file)
+      ret = 0
+    else
+      base_name = File.basename(working_dir)
+      out = log_and_time("taring up backup file #{backup_file}") do
+        # tar up the combined directory
+        `tar czf #{backup_file} -C #{@backup_dir} #{base_name}`
+      end
+      ret=$?
+    end
+    if ret.to_i != 0
+      log("error taring file : #{out}")
+      exit 1
+    end
+    if File.exists?(backup_file)
+      log("cleaning up working dir #{@working_dir}")
+      FileUtils.rm_rf(working_dir)
+    else
+      log("backup file #{backup_file} was not created properly")
+      return false
+    end
+    store_backup(storage_name, backup_file)
+    taken = Time.now - now
+    log("time taken to dump backup #{taken}")
+  end
+
+  def get_paths(settings = {}, append = nil)
+    append = settings[:append] if append.nil?
+    prepend = settings[:prepend]
+    now = Time.now
+    storage_name = "#{::Rails.env}_latest"
+    format = settings.fetch(:date_format, '%Y-%m-%d-%H')
+    date = now.strftime(format)
+    storage_name += "_#{append}" if append
+    base_name = "#{prepend}#{date}-#{::Rails.env}"
+    base_name += "-#{append}" if append
+    if custom?
+      ext = '.dump.gz'
+    else
+      ext = '.tar.gz'
+    end
+    storage_name += ext
+    back_name = "#{base_name}#{ext}"
+    working_dir = File.join(@backup_dir, base_name)
+    Dir.mkdir(working_dir) if !File.exists?(working_dir)
+    backup_file = File.join(@backup_dir, back_name)
+    [storage_name, working_dir, backup_file]
+  end
+
+  def custom?
+    false
+  end
+  
+  def store_backup(storage_name, backup_file)
+    if @save
+      # send file to s3
+      if @send_to_s3
+        log("sending backup file to s3")
+        send_file_to_s3(backup_file)
+      end
+      if @send_to_shared
+        if @use_scp
+          # maintain the name so we can send it to the storage properly
+          back_name = File.basename(backup_file)
+          shared_file = File.join(@shared_dir, back_name)
+          log("scping backup file to #{shared_file}")
+          `/usr/bin/scp #{backup_file} #{@backup_user}@#{@backup_machine}:#{shared_file}`
+        else
+          shared_file = File.join(@shared_dir, storage_name)
+          log("copying backup file to #{shared_file}")
+          FileUtils.copy_file(backup_file, shared_file)
+        end
+      end
+      cleanup_backups
+      cleanup_shared
+    else
+      log("skip save")
+    end
+  end
+
+  def shared_dst(create_if_missing = false)
+    return nil if !File.exists?(@shared_dir)
+    dst = File.join(@shared_dir, "#{::Rails.env}_daily")
+    Dir.mkdir(dst) if !File.exists?(dst) && create_if_missing
+    dst
+  end
+
+  def cleanup_backups(age = nil)
+    return false if !File.exists?(@backup_dir)
+    age = @cleanup_backups_age if age.blank?
+
+    clean_dir = @backup_dir
+    cleanup_dir(clean_dir, age) do |file|
+      if @move_backups
+        dst_file = shared_dst(true)
+        next if dst_file.blank? || !File.exists?(dst_file)
+        FileUtils.mv(file, dst_file)
+      else
+        File.delete(file)
+      end
+    end
+  end
+
+  def cleanup_shared(age = nil)
+    clean_dir = shared_dst
+    return false if clean_dir.nil? || !File.exists?(clean_dir)
+    age = @clean_shared_age if age.blank?
+    cleanup_dir(clean_dir, age) do |file|
+      File.delete(file)
+    end
+  end
+
+  def cleanup_dir(clean_dir, age, display = true)
+    cleaned = 0
+    total = 0
+    Dir["#{clean_dir}/*.tar.gz"].each do |file|
+      stat = File.stat(file)
+      if stat.mtime < age.to_i.days.ago
+        yield file
+        cleaned += stat.size
+        total += 1
+      end
+    end
+    log("cleaned up #{total} for #{cleaned} bytes in #{clean_dir}") if display
+    [total, cleaned]
+  end
+
+  def send_dir_to_s3
+    Dir.foreach(@backup_dir) do |f|
+      next if f[0] == ?.
+      file = File.join(backup_dir, f)
+      send_file_to_s3(file)
+    end
+  end
+
+  def send_file_to_s3(backup_file)
+    if @use_code
+      `/usr/bin/python #{@code_dir}/S3Storage.py upload #{backup_file}`
+      return true
+    end
+    bucket = 'twistage-backup'
+    # first check if backup file exists on s3, and then upload it
+    name = File.basename(backup_file)
+    s3 = S3Storage.default_connection
+    files = s3.list_bucket({:prefix => name}, bucket)
+    size = File.size(backup_file)
+    amt = 5
+    if files.size == 0
+      amt.times do |x|
+        log("uploading #{backup_file} to s3")
+        begin
+          start = Time.now
+          s3.put_file(backup_file, bucket)
+          taken = Time.now - start
+          mb = (size / 1.megabyte) / taken
+          log("file uploaded in #{taken} seconds (#{mb} MB/s)")
+          return true
+        rescue StandardError => err
+          if x+1 == amt
+            log("could not upload file #{backup_file} : #{err.to_s}")
+            return false
+          end
+          log("could not send file #{err.to_s}, retrying (#{x+1} of #{amt})")
+          sleep(15)
+        end
+      end
+    else
+      log("#{backup_file} already exists in s3")
+    end
+    true
+  end
+end

data/lib/backup_utility/file.rb

@@ -0,0 +1,98 @@
+class BackupUtility::File < BackupUtility::Base
+
+  def self.init(type)
+    save = ENV['save'] != 'false'
+    use_code = false
+    # don't send to shared, since we're going do it on "cleanup"
+    if ::Rails.env == 'production'
+      send_to_s3 = true
+    else
+      send_to_s3 = false
+    end
+    use_code = send_to_s3
+    settings = build_settings(type)
+    return [nil, nil] if settings.nil?
+    db_info = {:save => save, 
+      :send_to_s3 => send_to_s3,
+      :send_to_shared => false,
+      :use_code => use_code,
+      :shared_dir => "/mnt/shared_storage2/backups/#{type}",
+      :backup_dir => "/home/twist/backups/#{type}",
+      :clean_shared_age => ENV['clean_shared_age']
+    }
+    [BackupUtility::File.new(db_info), settings]
+  end
+
+  def self.build_settings(type)
+    age = (ENV['age'] || 5).to_i
+    if type == :custom
+      src_dir = ENV['custom_dir']
+      label = ENV['custom_label']
+      if src_dir.nil? || label.nil?
+        puts "missing custom dir or label"
+        return nil
+      end
+      type = label.to_sym
+    else
+      type = type.to_sym
+      map = {:logs => '/home/twist/stats_prod/shared/nginx_logs/archive'
+      }
+      if map.has_key?(type)
+        src_dir = map[type]
+      else
+        # check if the type exists on disk
+        src_dir = "/home/twist/stats_prod/shared/log/#{type}_dumps"
+      end      
+      if src_dir.blank? || !File.exists?(src_dir)
+        puts "could not find directory to backup for #{type} (src dir : #{src_dir})"
+        return nil
+      end
+    end
+    {:type => type, :src_dir => src_dir, :age => age.days.ago, :date_format => '%Y-%m-%d'}
+  end
+
+  def initialize(backup_info)
+    super(backup_info)
+  end
+
+  def get_paths(settings = {}, append = nil)
+    to_backup = find_dir(settings)
+    return [nil, nil, nil] if to_backup.nil?
+    date = File.basename(to_backup)
+    type = settings[:type]
+    base_name = "stats_#{type}_#{date}.tar.gz"
+    backup_file = File.join(@backup_dir, base_name)
+    working_dir = File.join(@backup_dir, date)
+    settings[:to_backup] = to_backup
+    [nil, working_dir, backup_file]
+  end
+
+  def find_dir(settings)
+    src_dir = settings[:src_dir]
+    return nil if src_dir.blank? || !File.exists?(src_dir)
+    age = settings[:age]
+    to_backup = nil
+    Dir.entries(src_dir).each do |name|
+      next if name[0] == ?.
+      dir = File.join(src_dir, name)
+      next if !File.directory?(dir)
+      mtime = File.mtime(dir)
+      if mtime < age
+        to_backup = dir
+        puts "found #{dir} to backup"
+        break
+      end
+    end
+    to_backup
+  end
+
+  def perform_dump(settings, working_dir)
+    return false if working_dir.blank?
+    # move everything in the to_backup dir to the working_dir, and then remove it
+    to_backup = settings[:to_backup]
+    return false if to_backup.blank?
+    `mv #{to_backup} #{@backup_dir}`
+    return false if File.exists?(to_backup)
+    true
+  end  
+end

data/lib/backup_utility/mongo.rb

@@ -0,0 +1,37 @@
+class BackupUtility::Mongo < BackupUtility::Base
+
+  def self.init(include_env = true)
+    save = ENV['save'] != 'false'
+    if ::Rails.env == 'staging'
+      send_to_s3 = false
+      send_to_shared = true
+    elsif ::Rails.env == 'production'
+      send_to_s3 = true
+      send_to_shared = true
+    elsif ::Rails.env == 'development'
+      send_to_s3 = false
+      send_to_shared = false
+    end
+    use_code = send_to_s3
+    db_info = {:backup_dir => ENV['backup_dir'], 
+      :save => save, 
+      :send_to_s3 => send_to_s3,
+      :send_to_shared => send_to_shared,
+      :use_code => use_code
+    }
+    BackupUtility::Mongo.new(db_info)
+  end
+  
+  def perform_dump(settings, working_dir)
+    type = settings[:server_type] || 'data_store'
+    server = MongoServer.select(type)
+    return false if server.blank?
+    db_names = settings[:db_names] || 'videos_day,tracks_day,ref'
+    db_names = db_names.split(',')
+    db_names.each do |db_name|
+      puts "dumping mongo db #{db_name} to #{working_dir}"
+      server.dump(working_dir, db_name)
+    end
+    true
+  end
+end

data/lib/backup_utility/postgres.rb

@@ -0,0 +1,260 @@
+require 'fileutils'
+
+BIN='/usr/bin'
+
+class BackupUtility::Postgres < BackupUtility::Base
+
+  def self.init(append = 'twist', include_env = true)
+    save = ENV['save'] != 'false'
+    move_backups = true
+    use_scp = use_code = false
+    if ::Rails.env == 'staging'
+      pg_user = 'admin'
+      db_name = "#{append}_stage"
+      send_to_s3 = false
+      send_to_shared = true
+    elsif ::Rails.env == 'production'
+      pg_user = "#{append}_db"
+      db_name = "#{append}_prod"
+      send_to_s3 = true
+      send_to_shared = true
+      use_scp = use_code = append == 'stats'
+      # move the backup files to shared, but only if its not stats, since it does its own thing
+      move_backups = append != 'stats'
+    elsif ::Rails.env == 'development'
+      pg_user = ENV['pg_user'] || 'admin'
+      db_name = ENV['db_name'] || "#{append}_dev"
+      send_to_s3 = false
+      # always set to false
+      save = false
+      send_to_shared = false
+    end
+    db_info = {:db_name => db_name, :pg_user => pg_user, 
+      :backup_dir => ENV['backup_dir'], 
+      :save => save, 
+      :send_to_s3 => send_to_s3,
+      :send_to_shared => send_to_shared,
+      :backup_format => ENV['backup_format'],
+      :use_code => use_code,
+      :move_backups => move_backups,
+      :use_scp => use_scp
+    }
+    if include_env
+      # look for these keys in the ENV variable
+      [:shared_dir, :backup_user, :backup_machine, :pg_host, :pg_port].each do |key|
+        val = ENV[key.to_s]
+        db_info[key] = val if val
+      end
+    end
+    BackupUtility::Postgres.new(db_info)
+  end
+
+  def initialize(db_info)
+    super(db_info)
+    @db_name = db_info.fetch(:db_name)
+    @pg_user = db_info.fetch(:pg_user, 'postgres')
+    config = ActiveRecord::Base.configurations[Rails.env]
+    if config
+      default_port = config['port']
+    else
+      default_port = nil
+    end
+    @pg_host = db_info.fetch(:pg_host, nil)
+    @pg_port = db_info.fetch(:pg_port, default_port)
+    @backup_user = db_info[:backup_user] || 'backup'
+    @backup_format = db_info[:backup_format] || 'plain'
+    raise "invalid format #{@backup_format}" if !['plain', 'custom', 'tar'].include?(@backup_format)
+    @save = db_info.fetch(:save, true)
+    @last_dst = nil
+  end
+
+  def get_paths(settings = {}, append = nil)
+    if !settings.fetch(:ignore, []).empty? || !settings.fetch(:only, []).empty?
+      name = "partial"
+      settings[:date_format] = '%Y-%m-%d-%H-%M'
+    else
+      name = "full"
+    end
+    name += "_#{append}" if append
+    super(settings, name)
+  end
+
+  def pg_dump_command
+    cmd = "#{BIN}/pg_dump -F#{@backup_format} -U #{@pg_user}"
+    cmd += " -h #{@pg_host}" if @pg_host
+    cmd += " -p #{@pg_port}" if @pg_port
+    cmd
+  end
+
+  def custom?
+    @backup_format == 'custom'
+  end
+
+  def dump_schema(working_dir)
+    _dump(working_dir, '--schema-only', '-schema')
+  end
+
+  def dump_data(working_dir, ignore = [])
+    ignore_string = custom? ? '' : '--data-only'
+    ignore.each {|i| ignore_string += ' -T '+i} if ignore.size > 0
+    _dump(working_dir, ignore_string)
+  end
+
+  def dump_data_only(working_dir, only = [])
+    only_string = custom? ? '' : '--data-only'
+    only.each {|o| only_string += ' -t '+o} if only.size > 0
+    label = "-table-" + only.join('-')
+    _dump(working_dir, only_string, label)
+  end
+
+  def _dump(working_dir, append_string = '', label='')
+    ext = custom? ? 'dump' : 'sql'
+    dst = File.join(working_dir, "#{@db_name}#{label}.#{ext}.gz")
+    log_and_time("dumping data (format #{@backup_format}) to #{dst}") do
+      cmd = "#{pg_dump_command} #{append_string} #{@db_name} | gzip > #{dst}"
+      @last_dst = dst
+      `#{cmd}`
+    end
+    File.exists?(dst)
+  end
+
+  def perform_dump(settings, working_dir)
+    # if the backup format is custom, do a full dump since we can do a bunch
+    # of stuff with pg_restore, which means we don't need the individual dumps
+    ignore = settings[:ignore] || []
+    only = settings[:only]
+    if custom?
+      if !only.nil? && !only.empty?
+        return dump_data_only(working_dir, only)
+      else
+        return dump_data(working_dir, ignore)
+      end
+    end
+    if !settings[:skip_schema]
+      if !dump_schema(working_dir)
+        log("failed to dump schema")
+        false
+      end
+    else
+      log("skipping schema dump")
+    end
+    separate = settings[:separate]
+    if separate
+      # first dump the database without any of the separate tables
+      log("dumping database without separate tables")
+      if !dump_data(working_dir, separate + ignore)
+        false
+      end
+      separate.each do |table|
+        log("dumping data only for table #{table}")
+        if !dump_data_only(working_dir, [table])
+          log("failed to dump individual table #{table}")
+          false
+        end
+      end
+      true
+    elsif !only.nil? && !only.empty?
+      dump_data_only(working_dir, only)
+    else
+      dump_data(working_dir, ignore)
+    end
+  end
+
+  def select_cmd(table, where, label)
+    "select * from #{table} where #{where} ORDER BY #{label} ASC"
+  end
+
+  def psql_dump_cmd(cmd, dump_file)
+    cmd = "#{BIN}/psql -c \"#{cmd}\" -F '"
+    cmd += 9.chr
+    cmd += "' -A -d #{@db_name} -U #{@pg_user} -t | gzip > #{dump_file}"
+    cmd
+  end
+
+  def dump_table_by_id(table, id, working_dir)
+    where = "id > #{id}"
+    cmd = select_cmd(table, where, 'id')
+    dump_file = File.join(working_dir, "dump_#{table}_#{id}.sql.gz")
+    dump_cmd = psql_dump_cmd(cmd, dump_file)
+    log("dumping table by id to #{dump_file}")
+    `#{dump_cmd}`
+    ret = $?
+    if !File.exists?(dump_file)
+      log("could not create dump #{dump_file}")
+      false
+    end
+    true
+  end
+
+  def determine_label(table)
+    if table =~ /hour/
+      'hour'
+    elsif table =~ /month/
+      'month'
+    else
+      'day'
+    end
+  end
+
+  def dump_daily_data(table, day, dump_dir)
+    format = '%m-%d-%Y'
+    start_str = day.strftime(format)
+    # determine the label by the table
+    label = determine_label(table)
+    end_str = (day + 1.day).strftime(format)
+    cmd = select_cmd(table, "#{label} >= '#{start_str}' AND #{label} < '#{end_str}'", label)
+    dump_name = "#{start_str}.sql.gz"
+    dump_file = File.join(dump_dir, dump_name)
+    # if the file exists, skip it as it appears to be already dumped
+    if File.exists?(dump_file)
+      log("dump file #{dump_file} already created, skipping")
+      return false
+    end
+    log("dumping stats to #{dump_file}")
+    dump_cmd = psql_dump_cmd(cmd, dump_file)
+    out = `#{dump_cmd}`
+    ret = $?
+    if !File.exists?(dump_file)
+      log("could not create dump #{dump_file}")
+      false
+    end
+    true
+  end
+
+  def dump_monthly_data(table, end_date = nil)
+    # the sql directory structure will look like this
+    # BACKUP_DIR
+    #  -> monthly-12-2009
+    #     -> hits_by_hour_video_geo
+    #        -> 12-1.2009.sql.gz
+    #        -> 12-2.2009.sql.gz
+    #        -> 12-3.2009.sql.gz
+    # then each directory under the monthly will be tarballed, 
+    # and uploaded to s3 with the name monthly-12-2009-hits_by_hour_video_geo.tar.gz
+    end_date = Time.now if end_date.blank?
+    # normalize to beginning of month
+    end_date = Time.utc(end_date.year, end_date.month, 1)
+    format = '%m-%Y'
+    start_day = (end_date - 1.month)
+    # normalize to the beginning of month
+    start_day = Time.utc(start_day.year, start_day.month, 1)
+    start_str = start_day.strftime(format)
+    table_dir = File.join(@backup_dir, "monthly-#{start_str}", table)
+    FileUtils.makedirs(table_dir)
+    # now iterate through all the days up till the end_date, and dump each file
+    while start_day < end_date
+      dump_daily_data(table, start_day, table_dir)
+      start_day += 1.day
+    end
+    # once we have the dir, tar ball the entire table dir
+    tar_ball = File.join(@backup_dir, "monthly-#{start_str}-#{table}.tar.gz")
+    `tar czvf #{tar_ball} #{table_dir}`
+    if File.exists?(tar_ball)
+      send_file_to_s3(tar_ball)
+      # now delete the tarball
+      File.delete(tar_ball)
+    else
+      log("tar ball does not exist #{tar_ball}")
+    end
+  end
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification 
+name: backup_utility
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 1
+  - 0
+  - 0
+  version: 1.0.0
+platform: ruby
+authors: 
+- Bruce Wang
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-11-02 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Create database dumps and store them in a remote location
+email: bwang@twistage.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/backup_utility.rb
+- lib/backup_utility/base.rb
+- lib/backup_utility/file.rb
+- lib/backup_utility/mongo.rb
+- lib/backup_utility/postgres.rb
+has_rdoc: true
+homepage: http://rubygems.org/gems/backup_utility
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Backup Utility
+test_files: []
+