automan 2.3.8 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cf346dc12fb2e3e0da31e57751b6158b64f58330
-  data.tar.gz: 9245efb48cab8a750fb4e2d44c8ede1d15652d45
+  metadata.gz: 556e8ef30f82380e1d0aa07bf033feb1dd2bf574
+  data.tar.gz: cf5aea391bdafeb455f5c9e9b0084797fced6881
 SHA512:
-  metadata.gz: f6604efc763560ca09bff13a965fe0c5a62000bde4f53af62c8ea17b2caf96ad693608fb843c258d8d74544d6e0601bdf4adec440b479ae7288fa5b6ba00b798
-  data.tar.gz: 34464ac37f622f24d4eb2f36c7b6af8a15610e08f85e07ccd5a9094de8f443162e0586f8a260d6d1da1a73e8900325213094c61b67e6f1317ee71be86262053b
+  metadata.gz: 9cac1683af9f00f2215e4430bc3787b1ada1f86d23a6de920b7e55cc800de2383aa7d8bbe4ab198474f134b79fdf5df006c88e8f8cb711f4f9d1cd4570379984
+  data.tar.gz: 90f61b9c7ccc3f75250e9c2c1efea0f405c31027ed9ae5c18aa6d42414a85cba396650d40630de830d6fe81299a5b97cdcce9e0fa9e03556be164c9365d3dcb3

data/lib/automan/cli/snapper.rb

@@ -7,28 +7,37 @@ module Automan::Cli
     desc "create", "create a snapshot"
 
     option :environment,
-    aliases: "-e",
-    desc: "environment of database to snapshot"
+      aliases: "-e",
+      desc: "environment of database to snapshot"
 
     option :database,
-    aliases: "-d",
-    desc: "name of the database to snapshot"
+      aliases: "-d",
+      desc: "name of the database to snapshot"
 
     option :name,
-    aliases: "-n",
-    desc: "what to name the snapshot"
+      aliases: "-n",
+      desc: "what to name the snapshot"
 
     option :prune,
-    aliases: "-p",
-    type: :boolean,
-    default: true,
-    desc: "make this snapshot prunable and delete other prunable snapshots older than 30 days"
+      aliases: "-p",
+      type: :boolean,
+      default: true,
+      desc: "make this snapshot prunable and delete other prunable snapshots older than 30 days"
+
+    option :type,
+      aliases: "-t",
+      desc: "type of snapshot. When pruning, only snapshots of the specified type will be deleted."
+
+    option :max_snapshots,
+      aliases: "-m",
+      desc: "Maximum number of snapshots of this type to retain",
+      type: :numeric
 
     option :wait_for_completion,
-    aliases: "-w",
-    type: :boolean,
-    default: false,
-    desc: "wait until snapshot is finished before exiting script"
+      aliases: "-w",
+      type: :boolean,
+      default: false,
+      desc: "wait until snapshot is finished before exiting script"
 
     def create
       if options[:database].nil? && options[:environment].nil?
@@ -37,6 +46,14 @@ module Automan::Cli
         exit 1
       end
 
+      if options[:prune]
+        if options[:type].nil? || options[:max_snapshots].nil?
+          puts "Must specify snapshot type and max snapshots to retain when pruning"
+          help "create"
+          exit 1
+        end
+      end
+
       aws_opts = options.dup
       aws_opts[:log_aws] = true
       s = Automan::RDS::Snapshot.new(aws_opts)
@@ -47,9 +64,9 @@ module Automan::Cli
     desc "delete", "delete a snapshot"
 
     option :name,
-    required: true,
-    aliases: "-n",
-    desc: "name of snapshot to delete"
+      required: true,
+      aliases: "-n",
+      desc: "name of snapshot to delete"
 
     def delete
       Automan::RDS::Snapshot.new(options).delete
@@ -58,12 +75,12 @@ module Automan::Cli
     desc "latest", "find the most recent snapshot"
 
     option :database,
-    aliases: "-d",
-    desc: "name of the database to snapshot"
+      aliases: "-d",
+      desc: "name of the database to snapshot"
 
     option :environment,
-    aliases: "-e",
-    desc: "environment of database to snapshot"
+      aliases: "-e",
+      desc: "environment of database to snapshot"
 
     def latest
       if options[:database].nil? && options[:environment].nil?
@@ -78,12 +95,12 @@ module Automan::Cli
     desc "count", "return the number of snapshots"
 
     option :environment,
-    aliases: "-e",
-    desc: "environment of database to snapshot"
+      aliases: "-e",
+      desc: "environment of database to snapshot"
 
     option :database,
-    aliases: "-d",
-    desc: "name of the database to snapshot"
+      aliases: "-d",
+      desc: "name of the database to snapshot"
 
     def count
       if options[:database].nil? && options[:environment].nil?
@@ -94,6 +111,6 @@ module Automan::Cli
 
       Automan::RDS::Snapshot.new(options).count_snapshots
     end
-     
+
   end
 end

data/lib/automan/rds/snapshot.rb

@@ -8,13 +8,14 @@ module Automan::RDS
                :name,
                :environment,
                :prune,
+               :type,
+               :max_snapshots,
                :wait_for_completion
 
-    attr_accessor :max_snapshots
-
     def initialize(options={})
       @prune = true
       @wait_for_completion = false
+      @max_snapshots = 50
       super
       @wait = Wait.new({
         delay: 30,
@@ -24,9 +25,8 @@ module Automan::RDS
         logger:   @logger
       })
 
-      if ENV['MAX_SNAPSHOTS'].nil?
-        @max_snapshots = 50
-      else
+      # DEPRECATED: use --max-snapshots instead
+      if ENV['MAX_SNAPSHOTS']
         @max_snapshots = ENV['MAX_SNAPSHOTS'].to_i
       end
     end
@@ -83,22 +83,12 @@ module Automan::RDS
 
       wait_until_database_available(db)
 
-      if snapshot_count >= max_snapshots
-        logger.info "Too many snapshots (>= #{max_snapshots}), deleting oldest prunable."
-        old = nil
-        AWS.memoize do
-          old = oldest_prunable_snapshot
-        end
-        logger.info "Deleting #{old.id}"
-        old.delete
-      end
-
       logger.info "Creating snapshot #{myname} for #{db.id}"
       snap = db.create_snapshot(myname)
 
       if prune == true
         logger.info "Setting snapshot to be prunable and tagging environment"
-        set_prunable_and_env(snap,environment)
+        set_prunable_and_env(snap)
       end
 
       if wait_for_completion == true
@@ -118,8 +108,7 @@ module Automan::RDS
     end
 
     def db_environment(db)
-      arn = db_arn(db)
-      return tags(arn)['Name']
+      return db_tags(db)['Name']
     end
 
     def default_snapshot_name(db)
@@ -149,10 +138,14 @@ module Automan::RDS
     end
 
     # tag with CanPrune
-    def set_prunable_and_env(snapshot,environment)
+    def set_prunable_and_env(snapshot)
       opts = {
         resource_name: snapshot_arn(snapshot),
-        tags: [ {key: 'CanPrune', value: 'yes'}, {key: 'Environment', value: environment} ]
+        tags: [
+          { key: 'CanPrune',    value: 'yes'},
+          { key: 'Environment', value: environment},
+          { key: 'Type',        value: type }
+        ]
       }
       response = rds.client.add_tags_to_resource opts
 
@@ -168,7 +161,17 @@ module Automan::RDS
       rds.db_snapshots[name].delete
     end
 
-    def tags(arn)
+    def db_tags(db)
+      arn = db_arn(db)
+      resource_tags(arn)
+    end
+
+    def snapshot_tags(snapshot)
+      arn = snapshot_arn(snapshot)
+      resource_tags(arn)
+    end
+
+    def resource_tags(arn)
       opts = {
         resource_name: arn
       }
@@ -185,67 +188,63 @@ module Automan::RDS
       result
     end
 
-    def can_prune?(snapshot)
-      tagged_can_prune?(snapshot) && available?(snapshot) && manual?(snapshot)
-    end
-
-    def is_env?(snapshot,environment)
-      tagged_env?(snapshot,environment) && available?(snapshot) && manual?(snapshot)
-    end
-
-    def tagged_env?(snapshot,environment)
-      arn = snapshot_arn(snapshot)
-      tags(arn)['Environment']  == environment
-    end
-
-    def tagged_can_prune?(snapshot)
-      arn = snapshot_arn(snapshot)
-      tags(arn)['CanPrune']  == 'yes'
-    end
-
     def available?(snapshot)
       snapshot.status == 'available'
     end
 
-    def manual?(snapshot)
-      snapshot.snapshot_type == 'manual'
-    end
-
-    # older than a month?
-    def too_old?(time)
-      time.utc < (Time.now.utc - 60*60*24*30)
-    end
-
     def get_all_snapshots
       rds.snapshots
     end
 
-    def get_all_env_snapshots
-      rds.db_instances[find_db.id].snapshots
-    end
-
-    def prunable_snapshots
-      snapshots = get_all_env_snapshots
-      snapshots.select { |s| can_prune?(s) }
+    # tags = {'mytagkey' => 'mytagvalue', ...}
+    def snapshot_has_tags?(snapshot, tags)
+      snap_tags = snapshot_tags(snapshot)
+      tags.each do |tk,tv|
+        if snap_tags[tk] != tv
+          #logger.info("qtags: #{tags.to_json} snap_tags: #{snap_tags.to_json}")
+          return false
+        end
+      end
+      true
     end
 
-    def oldest_prunable_snapshot
-      prunable_snapshots.sort_by { |s| s.created_at }.first
+    # tags = {'mytagkey' => 'mytagvalue', ...}
+    def snapshots_with_tags(tags)
+      rds.snapshots.select {|s| snapshot_has_tags?(s, tags)}
     end
 
+    #
+    # There is a big broken assumption here!
+    # We enumerate all snapshots of the *current* environment db
+    # What if that db is new and we have snapshots from previous
+    # dbs? The dev1 db is replaced every day!
+    # So prunable_snapshots will only return the list of snapshots
+    # from the *current* dev1 db and ignore all others!
+    #
     def prune_snapshots
       logger.info "Pruning old db snapshots"
 
+      tags = {
+        'Environment' => environment,
+        'Type'        => type,
+        'CanPrune'    => 'yes'
+      }
+      sorted_snaps = nil
       AWS.memoize do
-        prunable_snapshots.each do |snapshot|
+        sorted_snaps = snapshots_with_tags(tags).sort_by {|s| s.created_at}
+      end
 
-          timestamp = snapshot.created_at
-          snapshot_name = snapshot.db_snapshot_identifier
+      logger.info "snaps #{sorted_snaps.count} max #{max_snapshots}"
 
-          if too_old?(timestamp)
-            logger.info "Deleting #{snapshot_name} because it is too old."
-            snapshot.delete
-          end
+      if sorted_snaps.count >= max_snapshots
+        logger.info "Too many snapshots ( #{sorted_snaps.count} >= #{max_snapshots})."
+        number_to_delete = sorted_snaps.count - max_snapshots + 1
+        logger.info "deleting #{number_to_delete}."
+
+        condemned = sorted_snaps.take(number_to_delete)
+        condemned.each do |snap|
+          logger.info "Deleting #{snap.id}"
+          snap.delete
         end
       end
     end
@@ -265,16 +264,9 @@ module Automan::RDS
       log_options
       logger.info "Finding most recent snapshot for #{environment}"
 
-      allsnaps=get_all_snapshots
-
-      envsnaps=[]
-      allsnaps.each do |onesnap|
-        if is_env?(onesnap,environment)
-          envsnaps.push(onesnap)
-        end
-      end
+      tags = { 'Environment' => environment }
+      s = snapshots_with_tags(tags).sort_by {|s| s.created_at}.last
 
-      s=envsnaps.sort_by {|i| i.created_at }.last
       logger.info "Most recent snapshot is #{s.id}"
       s.id
     end

data/lib/automan/version.rb

@@ -1,3 +1,3 @@
 module Automan
-  VERSION = "2.3.8"
+  VERSION = "2.4.0"
 end

data/spec/rds/snapshot_spec.rb

@@ -58,36 +58,6 @@ describe Automan::RDS::Snapshot do
 
   end
 
-  describe '#tagged_can_prune?' do
-    subject() do
-      AWS.stub!
-      s = Automan::RDS::Snapshot.new
-      s.logger = Logger.new('/dev/null')
-      allow(s).to receive(:snapshot_arn)
-      s
-    end
-
-    it 'returns true if snapshot is tagged with CanPrune=yes' do
-      allow(subject).to receive(:tags).and_return( {'CanPrune' => 'yes'} )
-      expect(subject.tagged_can_prune?( double() )).to be_truthy
-    end
-
-    it 'returns false if snapshot is missing CanPrune tag' do
-      allow(subject).to receive(:tags).and_return( {} )
-      expect(subject.tagged_can_prune?( double() )).to be_falsey
-    end
-
-    it 'returns false if snapshot is tagged with CanPrune=nil' do
-      allow(subject).to receive(:tags).and_return( {'CanPrune' => nil} )
-      expect(subject.tagged_can_prune?( double() )).to be_falsey
-    end
-
-    it 'returns false if snapshot is tagged with CanPrune=foo' do
-      allow(subject).to receive(:tags).and_return( {'CanPrune' => 'foo'} )
-      expect(subject.tagged_can_prune?( double() )).to be_falsey
-    end
-  end
-
   describe '#available?' do
     subject() do
       AWS.stub!
@@ -106,45 +76,4 @@ describe Automan::RDS::Snapshot do
       expect(subject.available?(snap)).to be_falsey
     end
   end
-
-  describe '#manual?' do
-    let(:snap) { double }
-    subject() do
-      AWS.stub!
-      s = Automan::RDS::Snapshot.new
-      s.logger = Logger.new('/dev/null')
-      s
-    end
-
-    it 'returns true if type is "manual"' do
-      allow(snap).to receive(:snapshot_type).and_return('manual')
-      expect(subject.manual?(snap)).to be_truthy
-    end
-
-    it 'returns false if type is foo' do
-      allow(snap).to receive(:snapshot_type).and_return('foo')
-      expect(subject.manual?(snap)).to be_falsey
-    end
-  end
-
-  describe '#prunable_snapshots' do
-    let(:snap) { double }
-    subject() do
-      AWS.stub!
-      s = Automan::RDS::Snapshot.new
-      s.logger = Logger.new('/dev/null')
-      allow(s).to receive(:get_all_snapshots).and_return( [ snap ] )
-      s
-    end
-
-    it 'includes snapshots which can be pruned' do
-      allow(subject).to receive(:can_prune?).and_return(true)
-      expect(subject.prunable_snapshots).to include(snap)
-    end
-
-    it 'excludes snapshots which should not be pruned' do
-      allow(subject).to receive(:can_prune?).and_return(false)
-      expect(subject.prunable_snapshots).to_not include(snap)
-    end
-  end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: automan
 version: !ruby/object:Gem::Version
-  version: 2.3.8
+  version: 2.4.0
 platform: ruby
 authors:
 - Chris Chalfant
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-19 00:00:00.000000000 Z
+date: 2015-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -262,7 +262,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Automates common AWS ops