aebus 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+config/*
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in aebus.gemspec
+gemspec

data/README

@@ -0,0 +1,32 @@
+# Aebus - Automatic EC2 BackUp Software
+
+A small gem that allows you to easily automate EC2 backups, included purging of old backups and checking the backup status
+
+
+==Sample
+
+default:
+  access_key_id: <your_access_key_id>
+  secret_access_key: <your_secret_access_key>
+  zone: eu-west1
+  backups:
+    daily:
+      enabled: true
+      keep: 7
+      when: 0 3 * * *
+    weekly:
+      enabled: true
+      keep: 5
+      when: 0 3 * * 1
+    monthly:
+      enabled: true
+      keep: all
+      when: 0 3 1 * *
+
+
+vol-1234567:
+
+vol-7654321:
+  backups:
+    daily:
+      enabled: false

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/aebus.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "aebus/version"
+
+Gem::Specification.new do |s|
+  s.name        = "aebus"
+  s.version     = Aebus::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Marco Sandrini"]
+  s.email       = ["nessche@gmail.com"]
+  s.homepage    = "https://github.com/nessche/Aebus"
+  s.summary     = "Automated EC2 BackUp Software"
+  s.description = "A tool to automate snapshot management in EC2"
+
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency("commander", "~> 4.0.6")
+  s.add_dependency("parse-cron", ">= 0.1.1")
+  s.add_dependency("amazon-ec2", "~> 0.9.17")
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

data/bin/aebus

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'commander/import'
+require 'pathname'
+require_relative '../lib/aebus'
+
+DEFAULT_CONFIG_NAME = "./aebus.yaml"
+
+program :version, Aebus::VERSION
+program :description, 'Automatic EC2 BackUp Software'
+
+global_option('-c','--config FILE', 'The YAML file containing the backup configuration')
+
+default_command :help
+
+command :status do |c|
+  c.syntax = 'aebus status [options]'
+  c.summary = 'Checks for backup status'
+  c.description = <<-eos
+  Checks for the backup status of a given set of EC2 Volumes.
+  If no volume is specified in the command line, all volumes defined in the config files are checked
+  eos
+  c.example 'Checks the status of all volumes defined in the config file', 'aebus status'
+  c.example 'Checks the status of vol1 and vol2', 'aebus status vol1 vol2'
+
+  c.when_called do |args, options|
+    options.default \
+      :config => DEFAULT_CONFIG_NAME
+    raise ("Config file does not exist") unless FileTest.exist?(Pathname.new(options.config).realpath)
+    aebus = Aebus::Aebus.new
+    aebus.status(args,options)
+  end
+end
+
+command :backup do |c|
+  c.syntax = 'aebus backup [options]'
+  c.summary = 'Backs up  a set of EC2 Volumes'
+  c.description = <<-eos
+    Backs up a set of EC2 Volumes, according to the configuration file. If no volume is specified in the
+    command line, all volumes defined in the config file are backed up
+  eos
+  c.example 'Manually creates backup of vol1 and vol2', 'aebus backup --manual vol1 vol2'
+  c.example 'Create, if needed, backups for all volumes', 'aebus backup'
+  c.option '--manual', 'Starts a manual backup (always creates a snapshot for each volume)'
+  c.option '--[no-]purge', 'Do not purge expired backups, defaults to --purge'
+  c.when_called do |args, options|
+    options.default \
+      :config => DEFAULT_CONFIG_NAME,
+      :manual => false,
+      :purge => true
+    raise ("Config file does not exist") unless FileTest.exist?(Pathname.new(options.config).realpath)
+    aebus = Aebus::Aebus.new
+    aebus.backup(args,options)
+  end
+end
+

data/lib/aebus.rb

@@ -0,0 +1,208 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'AWS'
+require_relative 'config/config'
+require_relative 'aebus_zones'
+require_relative 'ec2/snapshot'
+
+module Aebus
+
+  VERSION = '0.0.1'
+
+  class Aebus
+
+    AWS_NAME_TAG = "Name"
+    AEBUS_TAG = "Aebus"
+
+    def status(args, options)
+      current_time_utc = Time.now.utc
+      config = Config::Config.new(File.join(File.dirname("."), options.config), current_time_utc)
+      @ec2 = AWS::EC2::Base.new(:access_key_id => config.defaults["access_key_id"],
+                               :secret_access_key => config.defaults["secret_access_key"],
+                               :server => zone_to_url(config.defaults["zone"]))
+
+      target_volumes = calculate_target_volumes(config, args)
+      snap_map = get_snapshots_map
+
+      target_volumes.each do |target|
+        volume = config.volumes[target]
+        tags = volume.backups_to_be_run(snap_map[target])
+        if (tags.count > 0) then
+          puts ("[INFO] Volume #{target} needs to be backed up. Tags: #{tags.join(',')}")
+        else
+          puts ("[INFO] Volume #{target} does not need to be backed up")
+        end
+
+        purgeable_snapshots =volume.purgeable_snapshots(snap_map[target])
+        puts ("[INFO] Volume #{target} has #{purgeable_snapshots.count} purgeable snapshot(s): #{purgeable_snapshots.inject([]){|x, snap| x << snap.id}.join(',')}")
+
+      end
+
+
+    end
+
+    def backup(args, options)
+
+      current_time_utc = Time.now.utc
+      config = Config::Config.new(File.join(File.dirname("."), options.config), current_time_utc)
+      @ec2 = AWS::EC2::Base.new(:access_key_id => config.defaults["access_key_id"],
+                               :secret_access_key => config.defaults["secret_access_key"],
+                               :server => zone_to_url(config.defaults["zone"]))
+
+      target_volumes = calculate_target_volumes(config, args)
+      if (options.manual) then
+
+        target_volumes.each do |volume|
+
+          backup_volume(volume, current_time_utc, [EC2::AEBUS_MANUAL_TAG])
+
+        end
+
+      else
+
+        snap_map = get_snapshots_map
+        target_volumes.each do |target|
+
+          volume = config.volumes[target]
+          tags = volume.backups_to_be_run(snap_map[target])
+          if (tags.count > 0) then
+            tags << EC2::AEBUS_AUTO_TAG
+            puts("[INFO] Creating backup for volume #{target} with tags #{tags.join(',')}")
+            backup_volume(target, current_time_utc, tags)
+          else
+            puts ("[INFO] Volume #{target} does not need to be backed up")
+          end
+
+        end
+
+        snap_map = get_snapshots_map # we reload the map since we may have created more snapshots
+        if (options.purge) then
+          target_volumes.each do |target|
+            volume = config.volumes[target]
+            purgeable_snapshots = volume.purgeable_snapshots(snap_map[target])
+            purgeable_snapshots.each {|snapshot| purge_snapshot(snapshot.id)}
+          end
+        else
+          puts("[INFO] Skipping purging phase")
+        end
+
+      end
+
+    end
+
+    def calculate_target_volumes(config, args)
+
+      result = config.volume_ids
+      if (args && (args.count > 0)) then
+        result &= args
+      end
+
+      result
+
+    end
+
+    def list_volumes
+      response = @ec2.describe_volumes
+      puts(response)
+    end
+
+
+    def zone_to_url(zone)
+      ZONES[zone]
+    end
+
+    def backup_volume(volume_id, current_time_utc, tags)
+      begin
+        volume_info = @ec2.describe_volumes(:volume_id => volume_id)
+
+      rescue AWS::Error => e
+        puts("[WARNING] Volume Id #{volume_id} not found")
+        return false
+      end
+
+      begin
+        puts(volume_info)
+        volume_tags = volume_info.volumeSet.item[0].tagSet.item
+        puts(volume_tags)
+
+        name_and_desc = Aebus.calculate_name_and_desc(volume_id, volume_tags, current_time_utc)
+        puts(name_and_desc)
+        create_response = @ec2.create_snapshot(:volume_id => volume_id, :description => name_and_desc[1])
+        puts(create_response)
+
+      rescue AWS::Error => e
+        puts("[ERROR] Volume Id #{volume_id} could not be backed up")
+        return false
+      end
+
+      begin
+
+        @ec2.create_tags(:resource_id => create_response.snapshotId,
+                         :tag => [{AWS_NAME_TAG => name_and_desc[0]}, {AEBUS_TAG => tags.join(',')}])
+      rescue AWS::Error => e
+        puts("[WARNING] Could not set tags to snapshot #{create_response.snapshotId}")
+        return false
+      end
+
+      puts("[INFO] Created snapshot #{create_response.snapshotId} for volume #{volume_id}");
+
+      return true
+
+    end
+
+
+    def self.calculate_name_and_desc(volume_id, tags, utc_time)
+
+      name = "backup_#{utc_time.strftime("%Y%m%d")}_#{volume_id}"
+      volume_name = volume_id
+      tags.each do |tag|
+        if tag["key"].eql?(AWS_NAME_TAG) then
+          volume_name = tag["value"]
+          break
+        end
+      end
+
+      description = "Backup for volume #{volume_name} taken at #{utc_time.strftime("%Y-%m-%d %H:%M:%S")}"
+
+      return [name, description]
+
+    end
+
+    def get_snapshots_map
+
+      response = @ec2.describe_snapshots(:owner => 'self')
+      snap_array = response.snapshotSet.item
+      result = Hash.new
+      snap_array.each do |snap|
+        snapshot = EC2::Snapshot.new(snap)
+        if (result.include?(snapshot.volume_id)) then
+          vol_array = result[snapshot.volume_id]
+          index = vol_array.index{ |s| snapshot.start_time > s.start_time}
+          index ||= vol_array.count
+          vol_array.insert(index, snapshot)
+        else
+          vol_array = Array.new
+          vol_array << snapshot
+          result.store(snapshot.volume_id, vol_array)
+        end
+      end
+      result
+
+    end
+
+    def purge_snapshot(snapshot_id)
+      begin
+        response = @ec2.delete_snapshot(:snapshot_id => snapshot_id)
+        if (response[return]) then
+          puts("[INFO] Purged snapshot #{snapshot_id}")
+        end
+      rescue AWS::Error => e
+        puts("[WARNING] Could not purge snapshot #{snapshot_id}")
+      end
+
+    end
+
+  end
+
+end

data/lib/aebus/version.rb

@@ -0,0 +1,3 @@
+module Aebus
+  VERSION = "0.0.1"
+end

data/lib/aebus_zones.rb

@@ -0,0 +1,11 @@
+module Aebus
+
+  ZONES = {
+
+      "eu-west1" =>  "eu-west-1.ec2.amazonaws.com",
+      "us-east-1" =>  "ec2.us-east-1.amazonaws.com",
+      "us-west-1" =>  "ec2.us-west-1.amazonaws.com",
+      "ap-southeast-1" => "ec2.ap-southeast-1.amazonaws.com"
+  }
+
+end

data/lib/config/config.rb

@@ -0,0 +1,60 @@
+require 'yaml'
+require 'cron_parser'
+require_relative 'volume'
+
+module Aebus
+
+  module Config
+
+    class Config
+
+      DEFAULT_STRING = 'default'
+
+      attr_reader :defaults, :volumes
+
+      def initialize(filename, current_time_utc)
+
+        yaml_root = YAML::load(File.open(filename))
+        raise "Cannot find configuration file" unless yaml_root
+
+        @defaults = yaml_root.delete(DEFAULT_STRING)
+        default_backups =  BackupSchedule.parse_backups_config(current_time_utc, @defaults["backups"])
+
+        @volumes = Hash.new
+        yaml_root.each_pair do |k, v|
+          @volumes[k] = Volume.new(current_time_utc, k, v, default_backups)
+        end
+
+      end
+
+      def volume_ids
+
+        result = Array.new
+        @volumes.each_key do |k|
+          result << k
+        end
+
+        result
+
+      end
+
+      def get_value_for_volume(volume_id, key)
+        result = nil
+        if (@volumes.include? volume_id) then
+          if (@volumes[volume_id].config.include? key) then
+            result = @volumes[volume_id].config[key]
+          else
+            result = @defaults[key]
+          end
+        end
+       result
+      end
+
+
+
+
+    end
+
+  end
+
+end

data/lib/config/volume.rb

@@ -0,0 +1,116 @@
+require 'cron_parser'
+
+module Aebus
+
+  module Config
+
+    KEEP_ALL = "all"
+
+    class BackupSchedule
+
+      attr_reader :label, :last_deadline, :next_deadline, :keep
+
+      def initialize (current_time_utc, label, backup_config)
+        @label = label
+        if (backup_config["enabled"]) then
+          calculate_deadlines(current_time_utc, backup_config["when"])
+        end
+        @keep = backup_config["keep"]
+
+      end
+
+      def calculate_deadlines(current_time_utc, when_string)
+        raise(ArgumentError, "when field cannot be empty if the backup is enabled") unless when_string
+
+        parser = CronParser.new (when_string)
+        @last_deadline = parser.last(current_time_utc)
+        @next_deadline = parser.next(current_time_utc)
+
+      end
+
+      def to_s
+        "Backup Schedule:  label => #{@label}  last_deadline => #{@last_deadline} next_deadline => #{@next_deadline}  keep => #{@keep}"
+      end
+
+      def self.parse_backups_config(current_time_utc, backups_config)
+
+        return nil unless backups_config
+
+        result = Hash.new
+
+        backups_config.each_pair do |key,value|
+           result.store(key, BackupSchedule.new(current_time_utc, key, value))
+        end
+
+        result
+
+      end
+
+    end
+
+    class Volume
+
+      attr_reader :id, :config
+
+      def initialize(current_time_utc, volume_id, config, default_backups)
+
+        @config = config
+        @id = volume_id
+        @backups = default_backups ? default_backups.dup : Hash.new
+        if (config && config["backups"]) then
+          @backups.merge(BackupSchedule.parse_backups_config(current_time_utc,config["backups"]))
+        end
+      end
+
+      def backups_to_be_run(snapshots)
+
+        result = Array.new
+
+        @backups.each_pair do |k,v|
+
+           result << k unless recent_backup?(k, snapshots, v.last_deadline)
+
+        end
+        result
+      end
+
+      def recent_backup?(label, snapshots, last_deadline)
+
+        snapshots.each do |snapshot|
+
+          if (snapshot.aebus_tags_include?(label) && (snapshot.start_time > last_deadline))
+            return true
+          end
+
+        end
+        false
+      end
+
+      def purgeable_snapshots(snapshots)
+        puts(snapshots.count)
+        removables = snapshots.select{|snapshot| snapshot.aebus_removable_snapshot?}
+        puts(removables.count)
+        available_backups = @backups.each_with_object({}) { | (k, v) , h | h[k] = v.keep}
+        removables.each do |snapshot|
+          snapshot.aebus_tags.each do |tag|
+            if (available_backups.include? tag) then
+              if (KEEP_ALL.eql?(available_backups[tag])) then
+                puts("Keeping snapshot #{snapshot.id} because of all on #{tag}")
+                snapshot.keep = true
+              elsif (available_backups[tag] > 0)  then
+                snapshot.keep = true
+                available_backups[tag] -= 1
+              end
+            end
+          end
+        end
+
+        removables.select{|snapshot| !snapshot.keep? }
+
+      end
+
+    end
+
+  end
+
+end

data/lib/ec2/snapshot.rb

@@ -0,0 +1,69 @@
+module Aebus
+
+  module EC2
+
+    AEBUS_TAG = "Aebus"
+    AEBUS_MANUAL_TAG = "manual"
+    AEBUS_KEEP_TAG = "keep"
+    AEBUS_AUTO_TAG = "auto"
+
+    class Snapshot
+
+
+
+      attr_reader :start_time, :volume_id, :id, :tags
+
+      def initialize(hash)
+
+        raise(ArgumentError,"hash cannot be nil") unless hash
+        @keep
+        @id = hash.snapshotId
+        @start_time = Time.parse(hash.startTime)
+        @volume_id = hash.volumeId
+        @tags = Hash.new
+        if (hash.tagSet) then
+          tag_array = hash.tagSet.item
+          tag_array.each do |tag|
+            @tags.store(tag["key"],tag["value"])
+          end
+        end
+      end
+
+      def to_s
+        "{snapshot_id => #{@id}, volume_id => #{@volume_id},  start_time => #{@start_time}, tags => #{@tags} "
+      end
+
+      def aebus_tags_include?(label)
+        if aebus_snapshot? then
+          aebus_tags = @tags[AEBUS_TAG].split(',')
+          return aebus_tags.include? label
+        end
+        false
+      end
+
+      def aebus_snapshot?
+        @tags.include?(AEBUS_TAG)
+      end
+
+      def aebus_removable_snapshot?
+        (aebus_tags & [AEBUS_MANUAL_TAG, AEBUS_KEEP_TAG]).count == 0
+      end
+
+      def aebus_tags
+        return nil unless aebus_snapshot?
+        @tags[AEBUS_TAG].split(',')
+      end
+
+      def keep= value
+        @keep = value
+      end
+
+      def keep?
+        @keep
+      end
+
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: aebus
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Marco Sandrini
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-10-05 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: commander
+  requirement: &70097059520900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.0.6
+  type: :runtime
+  prerelease: false
+  version_requirements: *70097059520900
+- !ruby/object:Gem::Dependency
+  name: parse-cron
+  requirement: &70097059520400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: *70097059520400
+- !ruby/object:Gem::Dependency
+  name: amazon-ec2
+  requirement: &70097059519940 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.17
+  type: :runtime
+  prerelease: false
+  version_requirements: *70097059519940
+description: A tool to automate snapshot management in EC2
+email:
+- nessche@gmail.com
+executables:
+- aebus
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README
+- Rakefile
+- aebus.gemspec
+- bin/aebus
+- lib/aebus.rb
+- lib/aebus/version.rb
+- lib/aebus_zones.rb
+- lib/config/config.rb
+- lib/config/volume.rb
+- lib/ec2/snapshot.rb
+homepage: https://github.com/nessche/Aebus
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Automated EC2 BackUp Software
+test_files: []