ebs_prune_snapshot 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.DS_Store

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ebs_prune_snapshot.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Dusty Doris
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.txt

@@ -0,0 +1,60 @@
+# EbsPruneSnapshot
+
+Manage your EBS snapshots on a daily, weekly, and monthly rotation schedule.
+
+Features
+ - Keep at least one snapshot of each volume, regardless of the rotation schedule.
+ - Saves the youngest snapshot of a particular day as a daily snapshot.
+ - Saves the oldest snapshot of any week as a weekly snapshot (Sundays).
+ - Saves the oldest snapshot of any 4 week period as a monthly snapshot (Sunday).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'ebs_prune_snapshot'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ebs_prune_snapshot
+
+## Usage
+
+AWS Access Key and AWS Secret Access Key are required.  These options can be
+provided through the command line or they may be stored in the environmental
+variables of AMAZON_ACCESS_KEY_ID and AMAZON_SECRET_ACCESS_KEY.  Any options
+provided through the command line will override the environmental variables.
+
+A daily, weekly, and monthly rotation are also required.  The monthly rotation
+is every 4 weeks, so provide 13 to go back a full year.
+
+If you only want to see what would happen if you ran the schedule, then pass
+the --dry-run (-n) argument.
+
+Usage: ebs_prune_snapshot [options]
+   eg: ebs_snapshot -d 7 -w 12 -m 14
+
+Required rotation configuration
+    -d, --daily DAILY                Number of daily snapshots to keep
+    -w, --weekly WEEKLY              Number of weekly snapshots to keep
+    -m, --monthly MONTHLY            Number of monthly snapshots to keep
+
+AWS Identifications, required unless environmental variables set
+    -k, --key KEY                    AWS Access Key (AMAZON_ACCESS_KEY_ID)
+    -s, --secret SECRET              AWS Secret Access Key (AMAZON_SECRET_ACCESS_KEY)
+
+Informational arguments
+    -n, --dry-run                    Describe what would be done (SAFE)
+    -h, --help                       Show Command Help
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,7 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+desc "Run tests"
+task :test do
+  sh "bundle exec test/test_ebs_prune_snapshot.rb"
+end

data/bin/ebs_prune_snapshot

@@ -0,0 +1,109 @@
+#!/usr/bin/env ruby
+require 'ebs_prune_snapshot'
+require 'optparse'
+
+def send_error(message)
+  puts @opts
+  puts ""
+  puts "ERROR: #{message}"
+  puts ""
+  exit(1)
+end
+
+@options = {}
+@opts = OptionParser.new
+
+@opts.banner = <<-EOD
+EbsPruneSnapshot
+
+Manage your EBS snapshots on a daily, weekly, and monthly rotation schedule.
+
+AWS Access Key and AWS Secret Access Key are required.  These options can be
+provided through the command line or they may be stored in the environmental
+variables of AMAZON_ACCESS_KEY_ID and AMAZON_SECRET_ACCESS_KEY.  Any options
+provided through the command line will override the environmental variables.
+
+A daily, weekly, and monthly rotation are also required.  The monthly rotation
+is every 4 weeks, so provide 13 to go back a full year.
+
+If you only want to see what would happen if you ran the schedule, then pass
+the --dry-run (-n) argument.
+
+Usage: ebs_prune_snapshot [options]
+   eg: ebs_snapshot -d 7 -w 12 -m 13"
+EOD
+
+ @opts.separator ""
+ @opts.separator "Required rotation configuration"
+
+ @opts.on(
+   "-d", "--daily DAILY", "Number of daily snapshots to keep"
+ ) { |value| @options[:daily]= value }
+
+ @opts.on(
+   "-w", "--weekly WEEKLY", "Number of weekly snapshots to keep"
+ ) { |value| @options[:weekly] = value }
+
+ @opts.on(
+   '-m', "--monthly MONTHLY", "Number of monthly snapshots to keep"
+ ) { |value| @options[:monthly] = value }
+
+@opts.separator ""
+@opts.separator "AWS Identifications, required unless environmental variables set"
+
+@opts.on(
+  "-k", "--key KEY", "AWS Access Key (AMAZON_ACCESS_KEY_ID)"
+) { |value| @options[:access_key] = value }
+
+@opts.on(
+  "-s", "--secret SECRET", "AWS Secret Access Key (AMAZON_SECRET_ACCESS_KEY)"
+) { |value| @options[:secret_key] = value }
+
+@opts.separator ""
+@opts.separator "Informational arguments"
+
+@opts.on(
+  "-n", "--dry-run", "Describe what would be done (DOES NOT delete snapshots)"
+) { |value| @options[:verbose] = true }
+
+@opts.on_tail("-h", "--help", "Show Command Help") do
+  puts @opts
+  puts ""
+  exit(1)
+end
+
+begin
+  @opts.parse!
+rescue StandardError => e
+  send_error e.message
+end
+
+@options[:access_key] ||= ENV['AMAZON_ACCESS_KEY_ID']
+@options[:secret_key] ||= ENV['AMAZON_SECRET_ACCESS_KEY']
+
+unless @options[:access_key] && @options[:secret_key]
+  send_error "Missing Arguments: --key and --secret are required
+ NOTE: AMAZON_ACCESS_KEY_ID and AMAZON_SECRET_ACCESS_KEY env variables are accepted"
+end
+
+unless @options[:daily] && @options[:weekly] && @options[:monthly]
+  send_error "Missing Arguments: --daily, --weekly and --monthly are required"
+end
+
+ebs = EbsPruneSnapshot::Base.new(
+  :aws => {
+    :access_key => @options[:access_key],
+    :secret_key => @options[:secret_key]
+  },
+  :rotation => {
+    :daily => @options[:daily].to_i,
+    :weekly => @options[:weekly].to_i,
+    :monthly => @options[:monthly].to_i
+  }
+)
+
+if @options[:verbose]
+  ebs.describe
+else
+  ebs.prune
+end

data/ebs_prune_snapshot.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/ebs_prune_snapshot/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Dusty Doris"]
+  gem.email         = ["dusty@doris.name"]
+  gem.description   = %q{Prune EBS snapshots}
+  gem.summary       = %q{Prune EBS snapshots on a daily, weekly, and monthly rotation}
+  gem.homepage      = "https://github.com/dusty/ebs_prune_snapshot"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "ebs_prune_snapshot"
+  gem.require_paths = ["lib"]
+  gem.version       = EbsPruneSnapshot::VERSION
+
+  gem.add_runtime_dependency('amazon-ec2')
+end

data/lib/ebs_prune_snapshot.rb

@@ -0,0 +1,249 @@
+require "ebs_prune_snapshot/version"
+require 'AWS'
+
+module EbsPruneSnapshot
+  class Base
+
+    attr_reader :rotation, :aws
+
+    def initialize(args={})
+      @aws = Aws.new(args[:aws])
+      @rotation = Rotation.new(args[:rotation])
+    end
+
+    def volumes
+      @volumes ||= begin
+        volumes = {}
+        aws.snapshots.each do |snapshot|
+          _snapshot = Snapshot.new(snapshot)
+          volumes[_snapshot.volume_id] ||= []
+          volumes[_snapshot.volume_id].push(_snapshot)
+        end
+        volumes
+      end
+    end
+
+    def volume(volume_id)
+      volumes[volume_id]
+    end
+
+    def describe
+      puts rotation
+      volumes.keys.each { |volume_id| describe_volume(volume_id) }
+    end
+
+    def prune
+      volumes.keys.each { |volume_id| prune_volume(volume_id) }
+    end
+
+    def describe_volume(volume_id)
+      analyzer = SnapshotAnalyzer.new(volume(volume_id), rotation)
+      puts "VOLUME: #{volume_id}"
+      analyzer.to_skip.each {|snapshot| puts   "  SKIP: #{snapshot.snapshot_id} : #{snapshot.date}"}
+      analyzer.to_save.each {|snapshot| puts   "  KEEP: #{snapshot.snapshot_id} : #{snapshot.date}"}
+      analyzer.to_delete.each {|snapshot| puts "  DELE: #{snapshot.snapshot_id} : #{snapshot.date}"}
+    end
+
+    def prune_volume(volume_id)
+      analyzer = SnapshotAnalyzer.new(volume(volume_id), rotation)
+      analyzer.to_delete.each { |snapshot| aws.delete(snapshot) }
+    end
+
+  end
+
+
+  class Aws
+
+    attr_reader :access_key, :secret_key
+
+    def initialize(args={})
+      args ||= {}
+      @access_key = args[:access_key] || ENV['AMAZON_ACCESS_KEY_ID']
+      @secret_key = args[:secret_key] || ENV['AMAZON_SECRET_ACCESS_KEY']
+    end
+
+    def connection
+      @aws ||= ::AWS::EC2::Base.new(
+        :access_key_id => access_key, :secret_access_key => secret_key
+      )
+    end
+
+    def snapshots
+      snapshots = connection.describe_snapshots(:owner => 'self')['snapshotSet']
+      snapshots ? snapshots["item"] : []
+    end
+
+    def delete(snapshot)
+      connection.delete_snapshot(:snapshot_id => snapshot.snapshot_id)
+    end
+
+  end
+
+  class SnapshotAnalyzer
+
+    attr_reader :snapshots, :rotation, :to_skip
+
+    def initialize(snapshots, rotation)
+      @snapshots = snapshots.sort
+      @rotation  = rotation
+      @to_skip   = []
+      @to_delete = []
+      @to_save   = {}
+      analyze_snapshots
+    end
+
+    def to_save
+      @to_save.empty? ? [snapshots[-1]] : @to_save.values
+    end
+
+    def to_delete
+      snapshots - to_save - to_skip
+    end
+
+    def now
+      @now ||= Time.now.utc
+    end
+
+    protected
+
+    def save_newest_snapshot(snapshot, day)
+      @to_save[day_key(day)] = snapshot
+    end
+
+    def save_oldest_snapshot(snapshot, day)
+      @to_save[day_key(day)] ||= snapshot
+    end
+
+    def skip_snapshot(snapshot)
+      @to_skip.push(snapshot) unless @to_skip.include?(snapshot)
+    end
+
+    def analyze_snapshots
+      snapshots.each do |snapshot|
+        if snapshot.completed?
+          prune_daily(snapshot) || prune_weekly(snapshot) || prune_monthly(snapshot)
+        else
+          skip_snapshot(snapshot)
+        end
+      end
+      to_save.sort!
+      to_skip.sort!
+      to_delete.sort!
+    end
+
+    ## Save newest snapshot on a day
+    def prune_daily(snapshot)
+      catch :found do
+        (0..rotation.daily - 1).each do |count|
+          day = in_days(count)
+          if day_key(snapshot.created_at) == day_key(day)
+            save_newest_snapshot(snapshot, day)
+            throw :found, true
+          end
+        end
+        false
+      end
+    end
+
+    ## Save oldest snapshot on a week
+    def prune_weekly(snapshot)
+      catch :found do
+        (0..rotation.weekly - 1).each do |count|
+          week_beg = beginning_of_week(in_weeks(count))
+          week_end = end_of_week(in_weeks(count))
+          if snapshot.created_at >= week_beg && snapshot.created_at <= week_end
+            save_oldest_snapshot(snapshot, week_beg)
+            throw :found, true
+          end
+        end
+        false
+      end
+    end
+
+    ## Save oldest snapshot of every 4 weeks
+    def prune_monthly(snapshot)
+      catch :found do
+        (0..rotation.monthly - 1).each do |count|
+          month_beg = beginning_of_week(in_months(count))
+          month_end = end_of_week(in_months(count))
+          if snapshot.created_at >= month_beg && snapshot.created_at <= month_end
+            save_oldest_snapshot(snapshot, month_beg)
+            throw :found, true
+          end
+        end
+        false
+      end
+    end
+
+    def in_months(months)
+      in_weeks(months * 4)
+    end
+
+    def in_weeks(weeks)
+      in_days(weeks * 7)
+    end
+
+    def in_days(days)
+      now - (86400 * days)
+    end
+
+    def beginning_of_week(date)
+      days_from_sunday = date.wday
+      bow = (date - (86400 * days_from_sunday))
+      Time.utc(bow.year, bow.month, bow.day, 0, 0, 0)
+    end
+
+    def end_of_week(date)
+      days_till_saturday = (6 - date.wday)
+      eow = (date + (86400 * days_till_saturday))
+      Time.utc(eow.year, eow.month, eow.day, 23, 59, 59)
+    end
+
+    def day_key(day)
+      day.strftime("%Y%m%d")
+    end
+
+  end
+
+  class Snapshot
+
+    attr_reader :snapshot_id, :volume_id, :status, :created_at
+
+    def initialize(params={})
+      @snapshot_id = params['snapshotId']
+      @volume_id   = params['volumeId']
+      @status      = params['status']
+      @created_at  = Time.parse(params['startTime']).utc
+    end
+
+    def completed?
+      status == 'completed'
+    end
+
+    def <=>(other)
+      created_at <=> other.created_at
+    end
+
+    def date
+      created_at.strftime("%Y%m%d")
+    end
+
+  end
+
+  class Rotation
+
+    attr_reader :daily, :weekly, :monthly
+
+    def initialize(args={})
+      @daily   = args[:daily] ? args[:daily].to_i : 7
+      @weekly  = args[:weekly] ? args[:weekly].to_i : 12
+      @monthly = args[:monthly] ? args[:monthly].to_i : 14
+    end
+
+    def to_s
+      "Rotation: Daily #{daily}, Weekly #{weekly}, Monthly #{monthly}"
+    end
+
+  end
+
+end

data/lib/ebs_prune_snapshot/version.rb

@@ -0,0 +1,3 @@
+module EbsPruneSnapshot
+  VERSION = "0.0.1"
+end

data/test/test_ebs_prune_snapshot.rb

@@ -0,0 +1,164 @@
+#!/usr/bin/env ruby
+
+#require 'time'
+require 'ebs_prune_snapshot'
+require 'minitest/autorun'
+
+##
+# Generates snapshots that simulate amazon-ec describe snapshots
+# - 400 daily completed snapshots for 2 different volumes
+# - 1 daily pending snapshot for 2 volumes
+# - 1 really old single snapshot for a 3rd volume
+module EbsPruneSnapshot
+  class Tester
+
+    def self.snapshots
+      @snapshots ||= new.snapshots
+    end
+
+    def self.now
+      Time.utc(2012,5,1)
+    end
+
+    def generate_snapshot(snapshot_id, volume_id, start_time)
+      {
+        "snapshotId" => snapshot_id,
+        "volumeId" => volume_id,
+        "status" => 'completed',
+        "startTime" => start_time.xmlschema,
+        "progress" => '100%',
+        "ownerId" => '00000000',
+        "volumeSize" => '1',
+        "description" => "test #{snapshot_id}"
+      }
+    end
+
+    def initialize
+      @volumes  = ['vol-000000', 'vol-000001']
+      @now      = Tester.now
+      @days     = 400
+    end
+
+    def snapshots
+      @items ||= begin
+        items = []
+        count = 0
+        # generate snapshots for each day for the previous 400 days for each volume
+        @volumes[0..1].each do |volume_id|
+          (0..@days - 1).each do |day|
+            snapshot_id = "%06d" % count
+            start_time  = @now - (86400 * day)
+            items << generate_snapshot(snapshot_id, volume_id, start_time)
+            count += 1
+          end
+        end
+        # add pending snapshots
+        items << generate_snapshot("111111", @volumes[0], @now - 86400).update("status" => "pending")
+        items << generate_snapshot("222222", @volumes[1], @now - 86400).update("status" => "pending")
+
+        # add 2 really old snapshots under another volume
+        items << generate_snapshot("333333", @volumes[3], @now - 86400 * 7 * 60)
+        items << generate_snapshot("444444", @volumes[3], @now - 86400 * 7 * 90)
+      end
+    end
+
+    # Starting on 5/1/12, assuming a snapshot taken every day, with a 7/12/13 rotation
+    def self.expectations
+      day = [
+        Time.utc(2012,5,1), Time.utc(2012,4,30), Time.utc(2012,4,29), Time.utc(2012,4,28), Time.utc(2012,4,27),
+        Time.utc(2012,4,26), Time.utc(2012,4,25)
+      ]
+      week = [
+        Time.utc(2012,4,29), Time.utc(2012,4,22), Time.utc(2012,4,15), Time.utc(2012,4,8), Time.utc(2012,4,1),
+        Time.utc(2012,3,25), Time.utc(2012,3,18), Time.utc(2012,3,11), Time.utc(2012,3,4), Time.utc(2012,2,26),
+        Time.utc(2012,2,19), Time.utc(2012,2,12)
+      ]
+      month = [
+        Time.utc(2012,4,29), Time.utc(2012,4,1), Time.utc(2012,3,4), Time.utc(2012,2,5), Time.utc(2012,1,8),
+        Time.utc(2011,12,11), Time.utc(2011,11,13), Time.utc(2011,10,16), Time.utc(2011,9,18), Time.utc(2011,8,21),
+        Time.utc(2011,7,24), Time.utc(2011,6,26), Time.utc(2011,5,29), Time.utc(2011,5,1)
+      ]
+      (day + week + month).uniq.sort
+    end
+  end
+end
+
+##
+# Stub snapshots
+class EbsPruneSnapshot::Aws
+  def snapshots
+    EbsPruneSnapshot::Tester.snapshots
+  end
+  def delete(snapshot)
+    puts "running delete on #{snapshot.snapshot_id}"
+  end
+end
+class EbsPruneSnapshot::SnapshotAnalyzer
+  def now
+    EbsPruneSnapshot::Tester.now
+  end
+end
+
+
+class TestEbsPruneSnapshot < MiniTest::Unit::TestCase
+  def self.ebs
+    @ebs ||= EbsPruneSnapshot::Base.new(:rotation => {:daily => 7, :weekly => 12, :monthly => 14})
+  end
+
+  def setup
+    @ebs = self.class.ebs
+  end
+
+  def self.now
+    Time.utc(2012,5,1)
+  end
+
+  def in_days(days)
+    self.class.now - (86400 * days)
+  end
+
+  def test_sort_by_volume
+    assert_equal self.class.ebs.volumes.count, 3
+  end
+
+  def test_volume_analyzer
+    vol1 = @ebs.volumes[@ebs.volumes.keys[0]]
+    vol2 = @ebs.volumes[@ebs.volumes.keys[1]]
+    [vol1, vol2].each do |snapshots|
+      analyzer = EbsPruneSnapshot::SnapshotAnalyzer.new(snapshots, @ebs.rotation)
+      assert_equal analyzer.to_save.count, 29
+      assert_equal analyzer.to_skip.count, 1
+      assert_equal analyzer.to_delete.count, 371
+    end
+  end
+
+  def test_safety_mechanism
+    snapshots = @ebs.volumes[@ebs.volumes.keys[2]]
+    analyzer = EbsPruneSnapshot::SnapshotAnalyzer.new(snapshots, @ebs.rotation)
+    assert_equal analyzer.to_save.count, 1
+    assert_equal analyzer.to_skip.count, 0
+    assert_equal analyzer.to_delete.count, 1
+    assert_equal analyzer.to_save.first.snapshot_id, "333333"
+  end
+
+  def test_actual_expected_dates
+    vol1 = @ebs.volumes[@ebs.volumes.keys[0]]
+    vol2 = @ebs.volumes[@ebs.volumes.keys[1]]
+    [vol1, vol2].each do |snapshots|
+      analyzer = EbsPruneSnapshot::SnapshotAnalyzer.new(snapshots, @ebs.rotation)
+      assert_equal analyzer.to_save.map {|s| s.created_at} , EbsPruneSnapshot::Tester.expectations
+    end
+  end
+
+  def test_describe
+    skip
+    @ebs.describe
+  end
+
+  def test_prune
+    skip
+    @ebs.prune
+  end
+
+end
+

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: ebs_prune_snapshot
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Dusty Doris
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-05-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: amazon-ec2
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Prune EBS snapshots
+email:
+- dusty@doris.name
+executables:
+- ebs_prune_snapshot
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.txt
+- Rakefile
+- bin/ebs_prune_snapshot
+- ebs_prune_snapshot.gemspec
+- lib/ebs_prune_snapshot.rb
+- lib/ebs_prune_snapshot/version.rb
+- test/test_ebs_prune_snapshot.rb
+homepage: https://github.com/dusty/ebs_prune_snapshot
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.22
+signing_key: 
+specification_version: 3
+summary: Prune EBS snapshots on a daily, weekly, and monthly rotation
+test_files:
+- test/test_ebs_prune_snapshot.rb