builderator 0.3.10

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 48baf4bb78a9121f6268913550ac1cdf5157aa1c
+  data.tar.gz: 761c4a044264ad1109a534ab41a382076d2298bc
+SHA512:
+  metadata.gz: 988210a27ccc466c5b0d1fa5548aae42524043b6c31fe290cbbf27e2d13d42f094b1552662befa899c7b047906b60e9153e71afb88598b3d2598ae5c337b5bbd
+  data.tar.gz: 20a6c23658cd4e12f583319512b42609fe6aa5fb0f894d56d01283248839c9f97223aaf69a22c381d6f573de514e41a2619f5b7799c8e9706349783e37ff318b

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rubocop.yml

@@ -0,0 +1,16 @@
+AllCops:
+ Exclude:
+   - libraries/**/*
+   - spec/**/*
+   - metadata.rb
+
+Encoding:
+ Enabled: false
+LineLength:
+ Enabled: false
+HashSyntax:
+ Enabled: false
+RescueModifier:
+  Enabled: false
+MethodLength:
+  Max: 24

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in builderator.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 John Manero
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+# Builderator
+__Tools to make CI Packer builds awesome__
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'builderator'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install builderator
+
+## Usage
+
+Run `bundle exec thor build help`
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/builderator/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/Thorfile

@@ -0,0 +1 @@
+require 'thor-scmversion'

data/VERSION

@@ -0,0 +1 @@
+0.3.10

data/bin/build

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+unless ENV.include?('BUNDLE_GEMFILE')
+  puts ''
+  puts '   / \\'
+  puts '  | ^ |'
+  puts "  ಠ _ ಠ  It looks like you're trying to run Builderator outside of Bundler!"
+  puts '  || ||  This is aggressively discouraged! Please re-run your command'
+  puts '  || ||  prepended with `bundle exec`'
+  puts '  || ||'
+  puts '  |  |   See http://bundler.io/ for help setting up Bundler and creating a Gemfile'
+  puts '   \\/'
+  puts ''
+  exit 1
+end
+
+require_relative '../lib/builderator/tasks'
+Builderator::Tasks::CLI.start(ARGV)

data/builderator.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'builderator/metadata'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'builderator'
+  spec.version       = Builderator::VERSION
+  spec.authors       = ['John Manero']
+  spec.email         = ['jmanero@rapid7.com']
+  spec.summary       = 'Tools to make CI Packer builds awesome'
+  spec.description   = Builderator::DESCRIPTION
+  spec.homepage      = 'https://github.com/rapid7/builderator'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = Dir[File.join(Builderator::PATH, 'bin/*')].map { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(/^(test|spec|features)\//)
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'rake', '~> 10.0'
+
+  spec.add_dependency 'aws-sdk', '~> 2.0'
+  spec.add_dependency 'bundler', '~> 1.7.0'
+  spec.add_dependency 'berkshelf', '~> 3.2'
+  spec.add_dependency 'chef', '~> 12.0'
+  spec.add_dependency 'thor', '~> 0.19.0'
+  spec.add_dependency 'thor-scmversion', '1.7.0'
+end

data/docs/clean.md

@@ -0,0 +1,21 @@
+build-clean
+===========
+Tasks to delete/deregister abandoned EC2 resources
+
+### Options
+* `--commit` Execute cleanup task. Default behavior is to display resources that would be removed
+* `--filter KEY VALUE [KEY VALUE []]` Key/value pairs to filter resources. Valid keys include tags and native resource properties (See `describe` responses in the Ruby AWS-SDK)
+
+### Commands
+* `configs` Delete launch configurations that are not associated with an autoscaling group.
+
+* `images` Delete images that are not associated with a launch configuration, a running instance, or are tagged as the 'parent' of an image that qualifies for any of the previous three conditions. Additionally, a fixed number of images can be retained per ordered groups.
+
+  **Options**
+  * `--group-by KEY [KEY []]` Tags/properties to group images by for pruning
+  * `--sort-by KEY` Tag/property to sort grouped images on (Default: creation_date)
+  * `--keep N` Number of images in each group to keep (Default: 0)
+
+* `snapshots` Delete snapshots that are not associated with existing volumes or images.
+
+* `volumes` Delete volumes that are not attached to instances.

data/lib/builderator.rb

@@ -0,0 +1,4 @@
+require 'builderator/metadata'
+
+module Builderator
+end

data/lib/builderator/control/ami.rb

@@ -0,0 +1,64 @@
+require 'aws-sdk'
+require 'date'
+
+require_relative '../util'
+
+module Builderator
+  module Control
+    ##
+    # Find AMI IDs to use for sources
+    ##
+    module AMI
+      ## Account IDs of public image owners
+      module Owners
+        SELF = 'self'.freeze
+        UBUNTU = '099720109477'.freeze
+      end
+
+      ## Filter fields defined in http://docs.aws.amazon.com/sdkforruby/api/Aws/EC2/Builderator::Util.ec2.html#describe_images-instance_method
+      FILTERS = %w(architecture block-device-mapping.delete-on-termination
+                   block-device-mapping.device-name block-device-mapping.snapshot-id
+                   block-device-mapping.volume-size block-device-mapping.volume-type
+                   description hypervisor image-id image-type is-public kernel-id
+                   manifest-location name owner-alias owner-id platform product-code
+                   product-code.type ramdisk-id root-device-name root-device-type
+                   state state-reason-code state-reason-message virtualization-type).freeze
+
+      class << self
+        def search(filters = {})
+          [].tap do |images|
+            Builderator::Util.ec2.describe_images(search_options(filters)).each { |page| images.push(*page.images) }
+          end
+        end
+
+        def latest(filters)
+          search(filters).sort do |a, b|
+            DateTime.iso8601(b.creation_date) <=> DateTime.iso8601(a.creation_date)
+          end.first
+        end
+
+        private
+
+        def search_options(filters)
+          {}.tap do |options|
+            options[:image_ids] = Util.to_array(filters.delete(:image_id)) if filters.include?(:image_id)
+            options[:owners] = Util.to_array(filters.delete(:owner) { 'self' })
+
+            rfilters = [].tap do |f|
+              filters.each do |k, v|
+                next if v.nil?
+
+                f << {
+                  :name => FILTERS.include?(k.to_s) ? k : "tag:#{ k }",
+                  :values => Util.to_array(v)
+                }
+              end
+            end
+
+            options[:filters] = rfilters unless rfilters.empty?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/builderator/control/clean.rb

@@ -0,0 +1,134 @@
+require_relative '../model'
+require_relative '../util'
+
+module Builderator
+  module Control
+    ##
+    # Control logic for cleanup tasks
+    ##
+    module Clean
+      class << self
+
+        def options(arg = nil)
+          return @options unless arg.is_a?(Hash)
+
+          @options = arg.clone
+
+          Util.region(@options.delete('region'))
+          @commit = @options.delete('commit') { false }
+          @limit = @options.delete('limit') { true }
+
+          @options
+        end
+
+        def configs!
+          resources = Model.launch_configs.unused(options)
+
+          limit!(Model::LaunchConfigs, 'Cleanup Launch Configurations', resources, &Proc.new)
+          aborted!(&Proc.new)
+
+          resources.each do |l, _|
+            yield :remove, "Launch Configuration #{ l }", :red
+            Model.launch_configs.resources.delete(l)
+
+            next unless commit?
+            # puts "delete_launch_configuration(:launch_configuration_name => #{l})"
+            Util.asg.delete_launch_configuration(:launch_configuration_name => l)
+          end
+        rescue Aws::AutoScaling::Errors::ServiceError => e
+          exceptions << Util::AwsException.new('Cleanup Launch Configurations', e)
+          yield(*exceptions.last.status)
+        end
+
+        def images!
+          resources = Model.images.unused(options)
+
+          limit!(Model::Images, 'Cleanup Images', resources, &Proc.new)
+          aborted!(&Proc.new)
+
+          resources.each do |i, image|
+            yield :remove, "Image #{ i } (#{ image[:properties]['name'] })", :red
+            Model.images.resources.delete(i)
+
+            next unless commit?
+            puts "deregister_image(:image_id => #{i})"
+            # Util.ec2.deregister_image(:image_id => i)
+          end
+        rescue Aws::EC2::Errors::ServiceError => e
+          exceptions << Util::AwsException.new('Cleanup Images', e)
+          yield(*exceptions.last.status)
+        end
+
+        def snapshots!
+          resources = Model.snapshots.unused
+
+          limit!(Model::Snapshots, 'Cleanup Snapshots', resources, &Proc.new)
+          aborted!(&Proc.new)
+
+          resources.each do |s, _|
+            yield :remove, "Snapshot #{ s }", :red
+            Model.snapshots.resources.delete(s)
+
+            next unless commit?
+            puts "delete_snapshot(:snapshot_id => #{s})"
+            # Util.ec2.delete_snapshot(:snapshot_id => s)
+          end
+        rescue Aws::EC2::Errors::ServiceError => e
+          exceptions << Util::AwsException.new('Cleanup Snapshots', e)
+          yield(*exceptions.last.status)
+        end
+
+        def volumes!
+          resources = Model.volumes.unused
+
+          limit!(Model::Volumes, 'Cleanup Volumes', resources, &Proc.new)
+          aborted!(&Proc.new)
+
+          resources.each do |v, _|
+            yield :remove, "Volume #{ v }", :red
+            Model.volumes.resources.delete(v)
+
+            next unless commit?
+            puts "delete_volume(:volume_id => #{v})"
+            # Util.ec2.delete_volume(:volume_id => v)
+          end
+        rescue Aws::EC2::Errors::ServiceError => e
+          exceptions << Util::AwsException.new('Cleanup Volumes', e)
+          yield(*exceptions.last.status)
+        end
+
+        def commit?
+          @commit && !@abort
+        end
+
+        def limit?
+          @limit
+        end
+
+        def aborted?
+          @commit && @abort
+        end
+
+        def exceptions
+          @exceptions ||= []
+        end
+
+        private
+
+        def aborted!
+          yield :aborted, 'The following resources will NOT be removed because'\
+            ' safty constraints have not been met!', :yellow if aborted?
+        end
+
+        def limit!(klass, task, resources)
+          return unless limit? && (resources.size >= klass::LIMIT)
+
+          exceptions << Util::LimitException.new(klass, task, resources)
+          @abort = true
+
+          yield(*exceptions.last.status)
+        end
+      end
+    end
+  end
+end

data/lib/builderator/metadata.rb

@@ -0,0 +1,5 @@
+module Builderator
+  PATH = File.expand_path('../../..', __FILE__)
+  VERSION = IO.read(File.join(PATH, 'VERSION')) rescue '0.0.1'
+  DESCRIPTION = IO.read(File.join(PATH, 'README.md')) rescue 'README.md not found!'
+end

data/lib/builderator/model.rb

@@ -0,0 +1,46 @@
+module Builderator
+  module Model
+    ##
+    # Shared model interface
+    ##
+    class Base
+      attr_reader :resources
+      LIMIT = 4
+
+      def initialize(*args)
+        fetch(*args)
+      end
+
+      def fetch
+        @resources = {}
+      end
+
+      def find(filters = {})
+        Util.filter(resources, filters)
+      end
+
+      def select(set = [])
+        resources.select { |k, _| set.include?(k) }
+      end
+
+      def reject(set = [])
+        resources.reject { |k, _| set.include?(k) }
+      end
+
+      def in_use(options = {})
+        find(options.fetch('filters', {}))
+      end
+
+      def unused(options = {})
+        Util.filter(reject(in_use(options)), options.fetch('filters', {}))
+      end
+    end
+  end
+end
+
+require_relative './model/images'
+require_relative './model/instances'
+require_relative './model/launch_configs'
+require_relative './model/scaling_groups'
+require_relative './model/snapshots'
+require_relative './model/volumes'