loe-super-ebs-pruner-9000 1.0.0

data/.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+config.yml

data/Rakefile

@@ -0,0 +1,26 @@
+require 'rake/testtask'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "super-ebs-pruner-9000"
+    gemspec.summary = "Thins EBS volume snapshots."
+    gemspec.description = "Thins EBS volume snapshots."
+    gemspec.email = "andrew@andrewloe.com"
+    gemspec.homepage = "http://github.com/loe/super-ebs-pruner-9000"
+    gemspec.authors = ["W. Andrew Loe III"]
+    gemspec.add_dependency('right_aws', '>=1.10.0')
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+task :default => :test
+
+Rake::TestTask.new(:test) do |test|
+  test.test_files = FileList.new('test/**/test_*.rb') do |list|
+    list.exclude 'test/test_helper.rb'
+  end
+  test.libs << 'test'
+  test.verbose = true
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/bin/pruner

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require File.expand_path(File.dirname(__FILE__) + '/../lib/pruner')
+
+options = {}
+
+OptionParser.new do |opts|
+  opts.banner = 'Usage: pruner [options] vol-1 vol-2... providing no volumes will default to all volumes associated with the account.'
+  
+  opts.on('-i', '--aws-id ID', 'AWS ID') do |id|
+    options[:aws_id] = id
+  end
+  
+  opts.on('-k', '--aws-key KEY', 'AWS Secret Key') do |key|
+    options[:aws_key] = key
+  end
+  
+  options[:verbose] = true
+  opts.on('-v', '--[no-]verbose', 'Run verbosely') do |v|
+    options[:verbose] = v
+  end
+  
+  options[:live] = false
+  opts.on('-l', '--[no-]live', 'Run live') do |l|
+    options[:live] = l
+  end
+  
+  opts.on_tail('-h', '--help', 'Show this message') do
+    puts "Super Pruner 9000 - #{Pruner::VERSION}"
+    puts opts
+    exit
+  end
+  
+  begin
+    opts.parse!
+    
+    raise 'You must provide AWS credentials.' unless options[:aws_id] && options[:aws_key]
+  rescue Exception => e
+    print "ERROR: #{e}\n\n"
+    puts opts
+    exit
+  end
+end
+
+options[:volumes] = ARGV
+
+Pruner.new(options).prune!

data/lib/pruner.rb

@@ -0,0 +1,102 @@
+require 'right_aws'
+require 'activesupport'
+
+require File.expand_path(File.dirname(__FILE__) + '/pruner/version')
+require File.expand_path(File.dirname(__FILE__) + '/pruner/silence_ssl_warning')
+
+class Pruner
+  attr_reader :options
+  attr_accessor :ec2, :volumes, :all_snapshots, :old_snapshots
+  
+  NOW = Time.now
+  
+  HOURLY_AFTER_A_DAY =                 {:after => NOW - 1.week,
+                                        :before => NOW - 1.day,
+                                        :interval => 1.hour,
+                                        :name => 'Hourly After A Day'}
+  DAILY_AFTER_A_WEEK =                 {:after => NOW - 1.month,
+                                        :before => NOW - 1.week,
+                                        :interval => 24.hours,
+                                        :name => 'Daily After A Week'}
+  EVERY_OTHER_DAY_AFTER_A_MONTH =      {:after => NOW - 3.month,
+                                        :before => NOW - 1.month,
+                                        :interval => 48.hours,
+                                        :name => 'Every Other Day After A Month'}
+  WEEKLY_AFTER_A_QUARTER =             {:after => NOW - 2.years,
+                                        :before => NOW - 3.months,
+                                        :interval => 1.week,
+                                        :name => 'Weekly After A Quarter'}
+  EVERY_THREE_WEEKS_AFTER_TWO_YEARS =  {:after => NOW - 10.years,
+                                        :before => NOW - 2.years,
+                                        :interval => 3.weeks,
+                                        :name => 'Every Three Weeks After Two Years'}
+  
+  RULES = [HOURLY_AFTER_A_DAY, DAILY_AFTER_A_WEEK, EVERY_OTHER_DAY_AFTER_A_MONTH, WEEKLY_AFTER_A_QUARTER, EVERY_THREE_WEEKS_AFTER_TWO_YEARS]
+  
+  def initialize(options ={})
+    @options =        options
+    @old_snapshots =  []
+  end
+  
+  def prune!
+    apply_rules
+    remove_snapshots
+  end
+  
+  def apply_rules
+    RULES.each do |rule|
+      cached_size = old_snapshots.size if options[:verbose]
+      apply_rule(rule)
+      puts "#{rule[:name]}:  #{old_snapshots.size - cached_size}" if options[:verbose]
+    end
+  end
+  
+  def apply_rule(rule)
+    volumes.each do |volume|
+      
+      # Gather the snapshots that might fall in the rule's time window.
+      vulnerable_snaps = snapshots(volume).select { |snap| snap[:aws_started_at] > rule[:after] && snap[:aws_started_at] < rule[:before]}
+      
+      # Step across the rule's time window one interval at a time, keeping the last snapshot in that window.
+      window_start = rule[:before] - rule[:interval]
+      while window_start > rule[:after] do
+        # Gather snaps in the window
+        snaps_in_window = vulnerable_snaps.select { |snap| snap[:aws_started_at] >= window_start && snap[:aws_started_at] < window_start + rule[:interval]}
+        # The first one in the selection survives
+        keeper = snaps_in_window.pop
+        # Send the rest to die.
+        @old_snapshots += snaps_in_window
+        # Shrink the window
+        window_start -= rule[:interval]
+      end
+    end
+    
+    old_snapshots
+  end
+  
+  def remove_snapshots
+    puts "Removing #{old_snapshots.size} Snapshots:" if options[:verbose]
+    old_snapshots.each do |snap|
+      puts "  #{snap[:aws_id]} - #{snap[:aws_started_at]} (#{snap[:aws_volume_id]})" if options[:verbose]
+      ec2.delete_snapshot(snap[:aws_id]) if options[:live]
+    end
+    
+    old_snapshots
+  end
+  
+  def ec2
+    @ec2 ||= RightAws::Ec2.new(options[:aws_id], options[:aws_key])
+  end
+  
+  def volumes
+    @volumes ||= options[:volumes].empty? ? ec2.describe_volumes.map {|vol| vol[:aws_id] } : options[:volumes]
+  end
+  
+  def all_snapshots
+    @all_snapshots ||= ec2.describe_snapshots
+  end
+  
+  def snapshots(volume)
+    all_snapshots.select { |snap| volume == snap[:aws_volume_id] }
+  end
+end

data/lib/pruner/silence_ssl_warning.rb

@@ -0,0 +1,8 @@
+class Net::HTTP
+  alias_method :old_initialize, :initialize
+  def initialize(*args)
+    old_initialize(*args)
+    @ssl_context = OpenSSL::SSL::SSLContext.new
+    @ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  end
+end

data/lib/pruner/version.rb

@@ -0,0 +1,3 @@
+class Pruner
+  VERSION = '1.0.0'
+end

data/super-ebs-pruner-9000.gemspec

@@ -0,0 +1,46 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{super-ebs-pruner-9000}
+  s.version = "1.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["W. Andrew Loe III"]
+  s.date = %q{2009-07-24}
+  s.default_executable = %q{pruner}
+  s.description = %q{Thins EBS volume snapshots.}
+  s.email = %q{andrew@andrewloe.com}
+  s.executables = ["pruner"]
+  s.files = [
+    ".gitignore",
+     "Rakefile",
+     "VERSION",
+     "bin/pruner",
+     "lib/pruner.rb",
+     "lib/pruner/silence_ssl_warning.rb",
+     "lib/pruner/version.rb",
+     "super-ebs-pruner-9000.gemspec",
+     "test/test_pruner.rb"
+  ]
+  s.homepage = %q{http://github.com/loe/super-ebs-pruner-9000}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.4}
+  s.summary = %q{Thins EBS volume snapshots.}
+  s.test_files = [
+    "test/test_pruner.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<right_aws>, ["= 1.10.0"])
+    else
+      s.add_dependency(%q<right_aws>, ["= 1.10.0"])
+    end
+  else
+    s.add_dependency(%q<right_aws>, ["= 1.10.0"])
+  end
+end

data/test/test_pruner.rb

@@ -0,0 +1,205 @@
+require 'test/unit'
+require 'rubygems'
+require 'mocha'
+require 'pruner'
+
+class TestPruner < Test::Unit::TestCase
+  
+  NOW = Time.now
+  
+  # From RightAws::Ec2#describe_instances documentation.
+  def volumes_array
+    [{:aws_id => "vol-foo"},
+     {:aws_id => "vol-bar"},
+     {:aws_id => "vol-baz"}]
+  end
+  
+  # Generate twice hourly for yesterday
+  # Should result in 24 deleted snapshots.
+  def twice_hourly_yesterday(vol)
+    yesterday = NOW - 1.day
+    (1..48).map do |i|
+      {:aws_progress   => "100%",
+       :aws_status     => "completed",
+       :aws_id         => "snap-yesterday-#{i}-#{vol}",
+       :aws_volume_id  => vol, 
+       :aws_started_at => yesterday - i * 30.minutes}
+    end
+  end
+  
+  # Generate 2 times daily for last week.
+  # Should result in 7 deleted snapshots.
+  def twice_daily_last_week(vol)
+    last_week = NOW - 1.week
+    (1..14).map do |i|
+      {:aws_progress   => "100%",
+       :aws_status     => "completed",
+       :aws_id         => "snap-last-week-#{i}-#{vol}",
+       :aws_volume_id  => vol,
+       :aws_started_at => last_week - i * 12.hours}
+    end
+  end
+  
+  # Generate daily for last month.
+  # Should result in 15 deleted snapshots
+  def daily_last_month(vol)
+    last_month = NOW - 1.month
+    (1..30).map do |i|
+      {:aws_progress   => "100%",
+       :aws_status     => "completed",
+       :aws_id         => "snap-last-month-#{i}-#{vol}",
+       :aws_volume_id  => vol,
+       :aws_started_at => last_month - i * 1.day}
+    end
+  end
+  
+   # Generate 2 weekly for last quarter.
+   # Should result in 6 deleted snapshots.
+  def twice_weekly_last_quarter(vol)
+    last_quarter = NOW - 3.months
+    (1..12).map do |i|
+      {:aws_progress   => "100%",
+       :aws_status     => "completed",
+       :aws_id         => "snap-last-quarter-#{i}-#{vol}",
+       :aws_volume_id  => vol,
+       :aws_started_at => last_quarter - i * 3.days}
+    end
+  end
+  
+  # Generate weekly for last two years.
+  # Should result in 16 deleted snapshots.
+  def weekly_two_years_ago(vol)
+    two_years_ago = NOW - 2.years
+    (1..48).map do |i|
+      {:aws_progress   => "100%",
+       :aws_status     => "completed",
+       :aws_id         => "snap-two-years-ago-#{i}-#{vol}",
+       :aws_volume_id  => vol,
+       :aws_started_at => two_years_ago - i * 1.week}
+    end
+  end
+  
+  # Generate a set that we can prune from with known results.
+  # RULES = [HOURLY_AFTER_A_DAY, DAILY_AFTER_A_WEEK, EVERY_OTHER_DAY_AFTER_A_MONTH, WEEKLY_AFTER_A_QUARTER, EVERY_THREE_WEEKS_AFTER_TWO_YEARS]
+  def snapshots_array_foo
+    twice_hourly_yesterday('vol-foo') + twice_daily_last_week('vol-foo') + daily_last_month('vol-foo') + twice_weekly_last_quarter('vol-foo') + weekly_two_years_ago('vol-foo')
+  end
+  
+  def snapshots_array_bar
+    twice_hourly_yesterday('vol-bar') + twice_daily_last_week('vol-bar') + daily_last_month('vol-bar') + twice_weekly_last_quarter('vol-bar') + weekly_two_years_ago('vol-bar')
+  end
+   
+  def snapshots_array_baz
+    twice_hourly_yesterday('vol-baz') + twice_daily_last_week('vol-baz') + daily_last_month('vol-baz') + twice_weekly_last_quarter('vol-baz') + weekly_two_years_ago('vol-baz')
+  end
+  
+  def mock_aws
+    @mock_ec2 = mock()
+    RightAws::Ec2.stubs(:new).returns(@mock_ec2)
+  end
+  
+  def new_pruner
+    @pruner = Pruner.new({:volumes => [], :verbose => true, :live => true})
+  end
+  
+  def setup
+    mock_aws
+    new_pruner
+  end
+  
+  def teardown
+    @pruner = nil
+  end
+  
+  def test_no_window_overlap
+    windows = [twice_hourly_yesterday('vol-foo'), twice_daily_last_week('vol-foo'), daily_last_month('vol-foo'), twice_weekly_last_quarter('vol-foo'), weekly_two_years_ago('vol-foo')]
+    windows.each do |window|
+      assert_equal window & (windows.flatten - window), []
+    end
+  end
+  
+  def test_version
+    assert_not_nil Pruner::VERSION
+  end
+  
+  def test_ec2
+    assert_not_nil @pruner.ec2
+  end
+  
+  def test_volumes
+    @mock_ec2.expects(:describe_volumes).returns(volumes_array).once
+    assert_equal @pruner.volumes, ['vol-foo', 'vol-bar', 'vol-baz']
+  end
+  
+  def test_snapshots
+    @mock_ec2.expects(:describe_snapshots).returns(snapshots_array_foo).once
+    assert_equal @pruner.snapshots('vol-foo'), snapshots_array_foo
+  end
+  
+  def test_apply_rule_HOURLY_AFTER_A_DAY
+    @pruner.volumes = ['vol-foo']
+    @mock_ec2.expects(:describe_snapshots).returns(twice_hourly_yesterday('vol-foo'))
+    @pruner.apply_rule(Pruner::HOURLY_AFTER_A_DAY)
+    assert_equal @pruner.old_snapshots.size, 24
+  end
+  
+  def test_apply_rule_DAILY_AFTER_A_WEEK
+    @pruner.volumes = ['vol-foo']
+    @mock_ec2.expects(:describe_snapshots).returns(twice_daily_last_week('vol-foo'))
+    @pruner.apply_rule(Pruner::DAILY_AFTER_A_WEEK)
+    assert_equal @pruner.old_snapshots.size, 7
+  end
+  
+  def test_apply_rule_EVERY_OTHER_DAY_AFTER_A_MONTH
+    @pruner.volumes = ['vol-foo']
+    @mock_ec2.expects(:describe_snapshots).returns(daily_last_month('vol-foo'))
+    @pruner.apply_rule(Pruner::EVERY_OTHER_DAY_AFTER_A_MONTH)
+    assert_equal @pruner.old_snapshots.size, 15
+  end
+  
+  def test_apply_rule_WEEKLY_AFTER_A_QUARTER
+    @pruner.volumes = ['vol-foo']
+    @mock_ec2.expects(:describe_snapshots).returns(twice_weekly_last_quarter('vol-foo'))
+    @pruner.apply_rule(Pruner::WEEKLY_AFTER_A_QUARTER)
+    assert_equal @pruner.old_snapshots.size, 6
+  end
+  
+  def test_apply_rule_EVERY_THREE_WEEKS_AFTER_TWO_YEARS
+    @pruner.volumes = ['vol-foo']
+    @mock_ec2.expects(:describe_snapshots).returns(weekly_two_years_ago('vol-foo'))
+    @pruner.apply_rule(Pruner::EVERY_THREE_WEEKS_AFTER_TWO_YEARS)
+    assert_equal @pruner.old_snapshots.size, 32
+  end
+  
+  def test_apply_rules
+    @pruner.volumes = ['vol-foo']
+    @mock_ec2.expects(:describe_snapshots).returns(snapshots_array_foo)
+    @pruner.apply_rules
+    assert_equal @pruner.old_snapshots.size, 24 + 7 + 15 + 6 + 32
+  end
+  
+  def test_remove_snapshots
+    dead_snaps = 24 + 7 + 15 + 6 + 32
+    @pruner.volumes = ['vol-foo']
+    @mock_ec2.expects(:describe_snapshots).returns(snapshots_array_foo)
+    @mock_ec2.expects(:delete_snapshot).times(dead_snaps)
+    @pruner.apply_rules
+    @pruner.remove_snapshots
+  end
+  
+  def test_one_volume_prune!
+    dead_snaps = 24 + 7 + 15 + 6 + 32
+    @mock_ec2.expects(:describe_volumes).returns([{:aws_id => "vol-foo"}]).once
+    @mock_ec2.expects(:describe_snapshots).returns(snapshots_array_foo).once
+    @mock_ec2.expects(:delete_snapshot).times(dead_snaps)
+    @pruner.prune!
+  end
+  
+  def test_multiple_volumes_prune!
+    dead_snaps = (24 + 7 + 15 + 6 + 32) * 3 # one for each array!
+    @mock_ec2.expects(:describe_volumes).returns(volumes_array).once
+    @mock_ec2.expects(:describe_snapshots).returns(snapshots_array_foo + snapshots_array_bar + snapshots_array_baz).once
+    @mock_ec2.expects(:delete_snapshot).times(dead_snaps)
+    @pruner.prune!
+  end
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification 
+name: loe-super-ebs-pruner-9000
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+platform: ruby
+authors: 
+- W. Andrew Loe III
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-07-24 00:00:00 -07:00
+default_executable: pruner
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 1.10.0
+    version: 
+description: Thins EBS volume snapshots.
+email: andrew@andrewloe.com
+executables: 
+- pruner
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Rakefile
+- VERSION
+- bin/pruner
+- lib/pruner.rb
+- lib/pruner/silence_ssl_warning.rb
+- lib/pruner/version.rb
+- super-ebs-pruner-9000.gemspec
+- test/test_pruner.rb
+has_rdoc: false
+homepage: http://github.com/loe/super-ebs-pruner-9000
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Thins EBS volume snapshots.
+test_files: 
+- test/test_pruner.rb