aebus 0.0.2 → 0.0.3

data/.gitignore

@@ -1,2 +1,3 @@
 config/*
-Gemfile.lock
+Gemfile.lock
+pkg

data/bin/aebus

@@ -12,6 +12,7 @@ program :version, Aebus::VERSION
 program :description, 'Automatic EC2 BackUp Software'
 
 global_option('-c','--config FILE', 'The YAML file containing the backup configuration')
+global_option('-l', '--logfile FILE', 'A log file which will receive the output')
 
 default_command :help
 
@@ -28,8 +29,13 @@ command :status do |c|
   c.when_called do |args, options|
     options.default \
       :config => DEFAULT_CONFIG_NAME
-    raise ("Config file does not exist") unless FileTest.exist?(Pathname.new(options.config).realpath)
-    aebus = Aebus::Aebus.new
+    begin
+      FileTest.exist?(Pathname.new(options.config).realpath)
+    rescue Errno::ENOENT => e
+      puts ("Configuration file not found")
+      exit(2)
+    end
+    aebus = Aebus::Core.new
     aebus.status(args,options)
   end
 end
@@ -41,9 +47,9 @@ command :backup do |c|
     Backs up a set of EC2 Volumes, according to the configuration file. If no volume is specified in the
     command line, all volumes defined in the config file are backed up
   eos
-  c.example 'Manually creates backup of vol1 and vol2', 'aebus backup --manual vol1 vol2'
+  c.example 'Manually creates backup of vol1 and vol2', 'aebus backup -m vol1 vol2'
   c.example 'Create, if needed, backups for all volumes', 'aebus backup'
-  c.option '--manual', 'Starts a manual backup (always creates a snapshot for each volume)'
+  c.option '-m', '--manual', 'Starts a manual backup (always creates a snapshot for each volume)'
   c.option '--[no-]purge', 'Do not purge expired backups, defaults to --purge'
   c.when_called do |args, options|
     options.default \
@@ -51,7 +57,7 @@ command :backup do |c|
       :manual => false,
       :purge => true
     raise ("Config file does not exist") unless FileTest.exist?(Pathname.new(options.config).realpath)
-    aebus = Aebus::Aebus.new
+    aebus = Aebus::Core.new
     aebus.backup(args,options)
   end
 end

data/lib/aebus.rb

@@ -4,73 +4,133 @@ require 'rubygems'
 require 'AWS'
 require_relative 'config/config'
 require_relative 'aebus/version'
+require_relative 'aebus/logging'
+require_relative 'aebus/volume_status'
 require_relative 'ec2/zones'
 require_relative 'ec2/snapshot'
 
 module Aebus
 
-  class Aebus
+  class Core
+
+    include Logging
 
     AWS_NAME_TAG = "Name"
     AEBUS_TAG = "Aebus"
 
     def status(args, options)
-      current_time_utc = Time.now.utc
-      config = Config::Config.new(File.join(File.dirname("."), options.config), current_time_utc)
-      @ec2 = AWS::EC2::Base.new(:access_key_id => config.defaults["access_key_id"],
-                               :secret_access_key => config.defaults["secret_access_key"],
-                               :server => EC2::zone_to_url(config.defaults["zone"]))
+      @current_time_utc = Time.now.utc
+      init_logger options
+      logger.info("status check started at #{@current_time_utc}")
+
+      @config = Config::Config.new(File.join(File.dirname("."), options.config), @current_time_utc)
+      @ec2 = AWS::EC2::Base.new(:access_key_id => @config.defaults["access_key_id"],
+                                :secret_access_key => @config.defaults["secret_access_key"],
+                                :server => EC2::zone_to_url(@config.defaults["zone"]))
+
+
+
+      target_volumes = target_volumes(args)
+
+      abort("Configuration contains invalid volumes") unless validate_target_volumes(target_volumes)
+
+      status = check_status(target_volumes)
+
+      message = "status check completed - #{status[:total]} volume(s) checked,  #{status[:to_backup]} to be backed up, max delay detected #{status[:delay]}s, #{status[:to_purge]} snapshots to be purged"
+      logger.info message
+      puts message
 
-      target_volumes = calculate_target_volumes(config, args)
-      snap_map = get_snapshots_map
 
+
+    end
+
+    def check_status(target_volumes)
+      result = {}
+      result[:timestamp] = @current_time_utc
+      snap_map = get_snapshots_map
+      result[:volumes] = Array.new
+      to_backup = 0
+      to_purge = 0
       target_volumes.each do |target|
-        volume = config.volumes[target]
-        tags = volume.backups_to_be_run(snap_map[target])
-        if (tags.count > 0) then
-          puts ("[INFO] Volume #{target} needs to be backed up. Tags: #{tags.join(',')}")
+        vs = VolumeStatus.new(target)
+        volume = @config.volumes[target]
+        vs.last_backup = volume.last_backup
+        vs.next_backup = volume.next_backup
+        to_be_run = volume.backups_to_be_run(snap_map[target], @current_time_utc)
+
+        vs.delay = to_be_run[0]
+        vs.tags = to_be_run[1]
+
+        if (vs.needs_backup?) then
+          logger.info("Volume #{target} needs to be backed up. Tags: #{vs.tags.join(',')}, max delay #{vs.delay}")
+          to_backup += 1
         else
-          puts ("[INFO] Volume #{target} does not need to be backed up")
+          logger.info("Volume #{target} does not need to be backed up")
         end
 
-        purgeable_snapshots =volume.purgeable_snapshots(snap_map[target])
-        puts ("[INFO] Volume #{target} has #{purgeable_snapshots.count} purgeable snapshot(s): #{purgeable_snapshots.inject([]){|x, snap| x << snap.id}.join(',')}")
+        vs.purgeable_snapshot_ids = volume.purgeable_snapshot_ids(snap_map[target])
+        to_purge += vs.purgeable_snapshot_ids.count  if vs.purgeable_snapshot_ids
+        logger.info("Volume #{target} has #{vs.purgeable_snapshot_ids.count} purgeable snapshot(s): #{vs.purgeable_snapshot_ids.join(',')}")
 
-      end
+        result[:volumes] << vs
 
+      end
+      result[:to_backup] = to_backup
+      result[:to_purge] = to_purge
+      result[:delay] = result[:volumes].inject([0]) {|acc, vs| acc << vs.delay}.max
+      result[:total] = result[:volumes].count
+      result
 
     end
 
+
+
     def backup(args, options)
 
-      current_time_utc = Time.now.utc
-      config = Config::Config.new(File.join(File.dirname("."), options.config), current_time_utc)
-      @ec2 = AWS::EC2::Base.new(:access_key_id => config.defaults["access_key_id"],
-                               :secret_access_key => config.defaults["secret_access_key"],
-                               :server => EC2::zone_to_url(config.defaults["zone"]))
 
-      target_volumes = calculate_target_volumes(config, args)
+      backed_up = 0
+      max_delay = 0
+      purged = 0
+      to_purge = 0
+      to_backup = 0
+      @current_time_utc = Time.now.utc
+      @config = Config::Config.new(File.join(File.dirname("."), options.config), @current_time_utc)
+
+      init_logger options
+      logger.info("backup started at #{@current_time_utc}")
+
+      @ec2 = AWS::EC2::Base.new(:access_key_id => @config.defaults["access_key_id"],
+                                :secret_access_key => @config.defaults["secret_access_key"],
+                                :server => EC2::zone_to_url(@config.defaults["zone"]))
+
+      target_volumes = target_volumes(args)
       if (options.manual) then
 
         target_volumes.each do |volume|
-
-          backup_volume(volume, current_time_utc, [EC2::AEBUS_MANUAL_TAG])
+          to_backup += 1
+          break unless backup_volume(volume, [EC2::AEBUS_MANUAL_TAG])
+          backed_up += 1
 
         end
 
       else
 
         snap_map = get_snapshots_map
+
         target_volumes.each do |target|
 
-          volume = config.volumes[target]
-          tags = volume.backups_to_be_run(snap_map[target])
+          volume = @config.volumes[target]
+          to_be_run = volume.backups_to_be_run(snap_map[target], @current_time_utc)
+          max_delay = [max_delay, to_be_run[0]].max
+          tags = to_be_run[1]
           if (tags.count > 0) then
             tags << EC2::AEBUS_AUTO_TAG
-            puts("[INFO] Creating backup for volume #{target} with tags #{tags.join(',')}")
-            backup_volume(target, current_time_utc, tags)
+            logger.info("Creating backup for volume #{target} with tags #{tags.join(',')}, max delay #{max_delay}")
+            to_backup +=1
+            break unless backup_volume(target, tags)
+            backed_up += 1
           else
-            puts ("[INFO] Volume #{target} does not need to be backed up")
+            logger.info("Volume #{target} does not need to be backed up")
           end
 
         end
@@ -78,21 +138,29 @@ module Aebus
         snap_map = get_snapshots_map # we reload the map since we may have created more snapshots
         if (options.purge) then
           target_volumes.each do |target|
-            volume = config.volumes[target]
-            purgeable_snapshots = volume.purgeable_snapshots(snap_map[target])
-            purgeable_snapshots.each {|snapshot| purge_snapshot(snapshot.id)}
+            volume = @config.volumes[target]
+            purgeable_snapshot_ids = volume.purgeable_snapshot_ids(snap_map[target])
+            purgeable_snapshot_ids.each do |snapshot_id|
+              to_purge += 1
+              purged += 1 if purge_snapshot(snapshot_id)
+
+            end
           end
         else
-          puts("[INFO] Skipping purging phase")
+          logger.info("Skipping purging phase")
         end
 
       end
 
+      message = "Backup Completed at #{Time.now}. Checked #{target_volumes.count} volume(s), backed up #{backed_up}, max delay detected #{max_delay},  #{to_purge} purgeable snapshot(s), #{purged} purged"
+      logger.info(message)
+      puts(message)
+
     end
 
-    def calculate_target_volumes(config, args)
+    def target_volumes(args)
 
-      result = config.volume_ids
+      result = @config.volume_ids
       if (args && (args.count > 0)) then
         result &= args
       end
@@ -101,35 +169,31 @@ module Aebus
 
     end
 
-    def list_volumes
-      response = @ec2.describe_volumes
-      puts(response)
+    def init_logger(options)
+      Logging.log_to_file(options.logfile) unless options.logfile.nil?
     end
 
-
-
-
-    def backup_volume(volume_id, current_time_utc, tags)
+# backs up a given volume using the given time as part of the name and setting the given tags to the snapshot
+# @param volume_id [String] the id of the volume to be backed up
+# @param tags [Array] an array of String to be used as tags for the snapshot
+# @return [boolean] true if the backup was successful, false otherwise
+    def backup_volume(volume_id, tags)
       begin
         volume_info = @ec2.describe_volumes(:volume_id => volume_id)
 
       rescue AWS::Error => e
-        puts("[WARNING] Volume Id #{volume_id} not found")
+        logger.error("Volume Id #{volume_id} not found. Underlying message #{e.message}")
         return false
       end
 
       begin
-        puts(volume_info)
         volume_tags = volume_info.volumeSet.item[0].tagSet.item
-        puts(volume_tags)
 
-        name_and_desc = Aebus.calculate_name_and_desc(volume_id, volume_tags, current_time_utc)
-        puts(name_and_desc)
+        name_and_desc = Core.name_and_desc(volume_id, volume_tags, @current_time_utc)
         create_response = @ec2.create_snapshot(:volume_id => volume_id, :description => name_and_desc[1])
-        puts(create_response)
 
       rescue AWS::Error => e
-        puts("[ERROR] Volume Id #{volume_id} could not be backed up")
+        logger.error("Volume Id #{volume_id} could not be backed up. Underlying message #{e.message}")
         return false
       end
 
@@ -138,18 +202,22 @@ module Aebus
         @ec2.create_tags(:resource_id => create_response.snapshotId,
                          :tag => [{AWS_NAME_TAG => name_and_desc[0]}, {AEBUS_TAG => tags.join(',')}])
       rescue AWS::Error => e
-        puts("[WARNING] Could not set tags to snapshot #{create_response.snapshotId}")
+        logger.error("[WARNING] Could not set tags to snapshot #{create_response.snapshotId}. Underlying message #{e.message}")
         return false
       end
 
-      puts("[INFO] Created snapshot #{create_response.snapshotId} for volume #{volume_id}");
+      logger.info("Created snapshot #{create_response.snapshotId} for volume #{volume_id}")
 
-      return true
+      true
 
     end
 
-
-    def self.calculate_name_and_desc(volume_id, tags, utc_time)
+# calculates the name and the description to be set to a snapshot
+# @param volume_id [String] the id of the volume whose snapshot we are creating
+# @param tags [Array] the tags currently associated with the Volume
+# @param utc_time [Time] the UTC time at which the backup process started (used to generate the correct name)
+# @return [Array] an array in the form of [name, description]]
+    def self.name_and_desc(volume_id, tags, utc_time)
 
       name = "backup_#{utc_time.strftime("%Y%m%d")}_#{volume_id}"
       volume_name = volume_id
@@ -192,10 +260,26 @@ module Aebus
       begin
         response = @ec2.delete_snapshot(:snapshot_id => snapshot_id)
         if (response["return"]) then
-          puts("[INFO] Purged snapshot #{snapshot_id}")
+          logger.info("Purged snapshot #{snapshot_id}")
+          true
+        else
+          false
         end
       rescue AWS::Error => e
-        puts("[WARNING] Could not purge snapshot #{snapshot_id}")
+        logger.warn("Could not purge snapshot #{snapshot_id}; underlying message #{e.message}")
+        false
+      end
+
+    end
+
+    def validate_target_volumes(target_volumes)
+      begin
+        @ec2.describe_volumes(:volume_id => target_volumes)
+        logger.info("Target volumes validated")
+        true
+      rescue AWS::Error => e
+        logger.error("Target validation failed with message '#{e.message}' Check your configuration")
+        false
       end
 
     end

data/lib/aebus/logging.rb

@@ -0,0 +1,27 @@
+require 'logger'
+
+module Aebus
+
+  module Logging
+
+    def logger
+      Logging.logger
+    end
+
+    def self.logger
+      @logger ||= Logger.new(STDOUT)
+    end
+
+    def self.log_to_file(file)
+      begin
+        @logger = Logger.new(file, 'daily')
+      rescue Errno::EACCES => e
+        logger.warn("Could not create log file, '#{e.message}'. Defaulting to STDOUT")
+      end
+    end
+
+
+  end
+
+
+end

data/lib/aebus/version.rb

@@ -1,3 +1,3 @@
 module Aebus
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/aebus/volume_status.rb

@@ -0,0 +1,23 @@
+module Aebus
+
+  class VolumeStatus
+
+    attr_accessor :id, :last_backup, :next_backup, :delay, :purgeable_snapshot_ids, :tags
+
+    def initialize(volume_id)
+       @id = volume_id
+    end
+
+    def needs_backup?
+      (!@tags.nil? && (@tags.count > 0))
+    end
+
+    def to_s
+      "Volume: id => #{id}, :last_backup => #{last_backup}, next_backup=> #{next_backup}, needs_backup? => #{needs_backup?}, delay => #{delay}, tags => #{tags}, purgeable_snapshot => #{purgeable_snapshots}"
+
+
+    end
+
+  end
+
+end

data/lib/config/volume.rb

@@ -16,6 +16,8 @@ module Aebus
           calculate_deadlines(current_time_utc, backup_config["when"])
         end
         @keep = backup_config["keep"]
+        # we use Infinity to model the keep all
+        @keep = 1.0 / 0  if (@keep.nil? || @keep.eql?(KEEP_ALL))
 
       end
 
@@ -39,7 +41,7 @@ module Aebus
         result = Hash.new
 
         backups_config.each_pair do |key,value|
-           result.store(key, BackupSchedule.new(current_time_utc, key, value))
+          result.store(key, BackupSchedule.new(current_time_utc, key, value))
         end
 
         result
@@ -62,20 +64,23 @@ module Aebus
         end
       end
 
-      def backups_to_be_run(snapshots)
+      def backups_to_be_run(snapshots,current_time_utc)
 
         result = Array.new
-
+        max_delay = 0
         @backups.each_pair do |k,v|
 
-           result << k unless recent_backup?(k, snapshots, v.last_deadline)
+          unless recent_backup?(k, snapshots, v.last_deadline)
+            result << k
+            max_delay = [max_delay, current_time_utc.to_i - v.last_deadline.to_i].max
+          end
 
         end
-        result
+        [max_delay, result]
       end
 
       def recent_backup?(label, snapshots, last_deadline)
-
+        return false unless snapshots
         snapshots.each do |snapshot|
 
           if (snapshot.aebus_tags_include?(label) && (snapshot.start_time > last_deadline))
@@ -86,31 +91,40 @@ module Aebus
         false
       end
 
-      def purgeable_snapshots(snapshots)
-        puts(snapshots.count)
+
+      def purgeable_snapshot_ids(snapshots)
+        return [] unless snapshots
         removables = snapshots.select{|snapshot| snapshot.aebus_removable_snapshot?}
-        puts(removables.count)
         available_backups = @backups.each_with_object({}) { | (k, v) , h | h[k] = v.keep}
         removables.each do |snapshot|
           snapshot.aebus_tags.each do |tag|
-            if (available_backups.include? tag) then
-              if (KEEP_ALL.eql?(available_backups[tag])) then
-                puts("Keeping snapshot #{snapshot.id} because of all on #{tag}")
-                snapshot.keep = true
-              elsif (available_backups[tag] > 0)  then
+            if ((available_backups.include? tag) && (available_backups[tag] > 0))  then
                 snapshot.keep = true
                 available_backups[tag] -= 1
-              end
             end
           end
         end
 
-        removables.select{|snapshot| !snapshot.keep? }
+        removables.inject([]) do |acc, snapshot|
+          acc << snapshot.id unless snapshot.keep?
+          acc
+        end
+
+      end
+
+      def last_backup
+        @backups.values.map{|backup| backup.last_deadline}.max
+      end
 
+      def next_backup
+        @backups.values.map{|backup| backup.next_deadline}.min
       end
 
+
     end
 
+
+
   end
 
 end

data/lib/ec2/snapshot.rb

@@ -35,7 +35,6 @@ module Aebus
 
       def aebus_tags_include?(label)
         if aebus_snapshot? then
-          aebus_tags = @tags[AEBUS_TAG].split(',')
           return aebus_tags.include? label
         end
         false
@@ -46,11 +45,11 @@ module Aebus
       end
 
       def aebus_removable_snapshot?
+        return false unless aebus_snapshot?
         (aebus_tags & [AEBUS_MANUAL_TAG, AEBUS_KEEP_TAG]).count == 0
       end
 
       def aebus_tags
-        return nil unless aebus_snapshot?
         @tags[AEBUS_TAG].split(',')
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aebus
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-06 00:00:00.000000000Z
+date: 2011-11-28 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: commander
-  requirement: &70320653233040 !ruby/object:Gem::Requirement
+  requirement: &70278571512580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 4.0.6
   type: :runtime
   prerelease: false
-  version_requirements: *70320653233040
+  version_requirements: *70278571512580
 - !ruby/object:Gem::Dependency
   name: parse-cron
-  requirement: &70320653232540 !ruby/object:Gem::Requirement
+  requirement: &70278571512080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 0.1.1
   type: :runtime
   prerelease: false
-  version_requirements: *70320653232540
+  version_requirements: *70278571512080
 - !ruby/object:Gem::Dependency
   name: amazon-ec2
-  requirement: &70320653232080 !ruby/object:Gem::Requirement
+  requirement: &70278571511620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: 0.9.17
   type: :runtime
   prerelease: false
-  version_requirements: *70320653232080
+  version_requirements: *70278571511620
 description: A tool to automate snapshot management in EC2
 email:
 - nessche@gmail.com
@@ -59,7 +59,9 @@ files:
 - aebus.gemspec
 - bin/aebus
 - lib/aebus.rb
+- lib/aebus/logging.rb
 - lib/aebus/version.rb
+- lib/aebus/volume_status.rb
 - lib/config/config.rb
 - lib/config/volume.rb
 - lib/ec2/snapshot.rb