outliers 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 370c7538e31dc0846ae0de97c21b9eadc6cafe22
-  data.tar.gz: ecb6077e1e2451e3f7d19b58ab0d5ebedf673178
+  metadata.gz: ca2a95007c8c6b9d85847b7c857e8e5f54e72859
+  data.tar.gz: 273c267dc763814b17162cf394cc453e77ba2253
 SHA512:
-  metadata.gz: 5ef91692770636f84be52db5fa1989d7d9d3c54bda9fa206b60e63fc1b7186cfc3959c02287af1ae8b575ba6fe8a7f191d9d866bd43c4c15a394174ca485dacd
-  data.tar.gz: 8ab846d7b565ccaf4dfb4afaf377971e058dbc99f5d1fda119c57fe9a1d53a46dd7ca95d60eaa16acfe19ab2db18540b489ab8f1f8b1d8c73bd5edafeef28640
+  metadata.gz: efbb5c48979eafc1001695502ad892ae799517497b789e6b56d034b44e0e3cdf74fa40c950cf77e9fd329cd0afbe1fb4ab612497c6b0a33ab7d00e95c1b75cba
+  data.tar.gz: a2a0da57894add6de86c37a749d172a8174476b12f1d8665ebe68df9b11c31271bab95de1c1171f401cc4f8a2619e9726ad426f62b66914ec0e1b280f8aa74e4

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.2.0
+
+* Added filters
+* Added multiple verificaiton support
+* Remove all but AWS resources
+* Multiple bug fixes
+
 ## 0.1.1
 
 * Bug fix in evaluate CLI.

data/README.md

@@ -6,11 +6,17 @@ Outliers is a framework for verifying configuration of resources.
 
 ## Overview
 
-* Applications and teams rely on multiple service providers (AWS, Github, etc).
+* Applications and teams rely on multiple service providers (AWS, etc).
 * Providers deliver like resources with complex configuration (EC2 Instances, S3 Buckets, etc).
 * Resource configuration can be verified (launched from given AMI, contain private objects, etc).
+* The resources can be included or excluded by their ID (Instance ID, Object Key, etc).
+* Resources can be included in the list by matching a filter (Instance has tag 'x' with value 'y').
 * Those not passing verifications, are flagged as Outliers.
 
+## Requirements
+
+* Ruby 1.9.3 or greater.
+
 ## Installation
 
 Install the gem:
@@ -19,6 +25,8 @@ Install the gem:
 
 ## Setup
 
+**Currently Outliers only supports AWS**
+
 Create **~/outliers.yml** with a list of credentials in the following format:
 
     credential_name:
@@ -37,7 +45,9 @@ Multiple accounts can be specified, to add a prod and preprod AWS account:
       access_key_id: AAA
       secret_access_key: BBB
 
-Depending on the provider, different keys and values are required. For a list of providers:
+Depending on the provider, different keys and values are required.
+
+For a list of providers:
 
     outliers providers
 
@@ -77,6 +87,10 @@ To exclude resources that are known exceptions:
 
     outliers evaluate -c aws_prod -p aws_ec2 -r instance -e i-12345678
 
+Resources have attributes which can be used to filter target resources. To filter instances who have tag 'Name' equal to 'web'.
+
+    outliers evaluate -c aws_prod -p aws_ec2 -r instance -f 'tag=Name:web'
+
 ### DSL
 
 To run Outliers as a DSL
@@ -122,6 +136,16 @@ The DSL supports any valid Ruby code. To iterate over multiple regions:
       end
     end
 
+Evaluations can run multiple verifications. To validate instances are in a VPC, running and using a valid image:
+
+    evaluate do
+      connect 'aws_prod', provider: 'aws_ec2', region: 'us-west-1'
+      resources 'instance'
+      verify 'vpc'
+      verify 'running'
+      verify 'valid_image_id', image_ids: ['ami-12345678','ami-87654321']
+    end
+
 Evaluations can be given names to help identify Outliers in results.
 
     evaluate "validate_database_retention_period" do
@@ -171,6 +195,15 @@ Sometimes you want to exclude resources that are known exceptions, to exclude an
       verify 'valid_image_id', image_ids: ['ami-12345678','ami-87654321']
     end
 
+Resources have attributes which can be used to filter target resources. To filter instances who have tag 'Name' equal to 'web'.
+
+    evaluate do
+      connect 'aws_prod', provider: 'aws_ec2', region: 'us-west-1'
+      resources 'instance'
+      filter tag: 'Name:web'
+      verify 'valid_image_id', image_ids: ['ami-12345678','ami-87654321']
+    end
+
 ### Help
 
 For a list of providers and required credentials:

data/lib/outliers/cli/evaluate.rb

@@ -2,7 +2,7 @@ module Outliers
   module CLI
     class Evaluate
       def evaluate
-        @options = { arguments: [], exclude: [], credentials: [], target_resources: [] }
+        @options = { arguments: [], exclude: [], filter: [], credentials: [], target_resources: [] }
         @credentials = {}
 
         option_parser.parse!
@@ -15,6 +15,7 @@ module Outliers
         load_credentials
 
         @options[:parsed_arguments] = parse_arguments
+        @options[:parsed_filters] = parse_filters
 
         # Make options available global
         # Required to read by instance_eval in @run.evaluate 
@@ -25,6 +26,7 @@ module Outliers
             connect 'cli'
             resources @@options[:resource], @@options[:target_resources]
             exclude @@options[:exclude] if @@options[:exclude].any?
+            filter @@options[:parsed_filters] if @@options[:filter].any?
             verify @@options[:verification], @@options[:parsed_arguments]
           end
         rescue Outliers::Exceptions::Base => e
@@ -61,6 +63,19 @@ module Outliers
         @run.credentials = { 'cli' => credentials }
       end
       
+      def parse_filters
+        filters = {}
+
+        @options[:filter].each do |a|
+          key = a.split('=').first
+          value = a.split('=').last
+          filters.merge! key => value
+        end
+
+        filters
+      end
+
+      
       def parse_arguments
         arguments = {}
 
@@ -90,6 +105,10 @@ module Outliers
             @options[:exclude] << o
           end
 
+          opts.on("-f", "--fitler [FILTER]", "Equals seperated filter name and value (can be specified multiple times).") do |o|
+            @options[:filter] << o
+          end
+
           opts.on("-p", "--provider [PROVIDER]", "Provider of target resources.") do |o|
             @options[:provider] = o
           end

data/lib/outliers/cli/process.rb

@@ -22,8 +22,11 @@ module Outliers
         passed = @run.passed.count
         failed = @run.failed.count
 
+        @logger.info "Evaluations completed."
+
         @run.failed.each do |f|
-          @logger.info "Evaluation '#{f.description}' failed."
+          @logger.info "Evaluation '#{f.evaluation}' verification '#{f.verification}' of '#{f.resource}' failed."
+          @logger.debug "Failing resource IDs '#{f.failing_resources.map{|r| r.id}.join(', ')}'"
         end
 
         @logger.info "(#{failed} evaluations failed, #{passed} evaluations passed.)"

data/lib/outliers/cli/resources.rb

@@ -25,7 +25,10 @@ module Outliers
             name.slice! provider
             name[0] = ''
             puts name
-            r.verifications.each { |v| puts "  #{v[:name]}(#{v[:args]}) #{v[:description]}" }
+            puts "  Verifications:"
+            r.verifications.each { |v| puts "    #{v[:name]}(#{v[:args]}) #{v[:description]}" }
+            puts "  Filters:"
+            r.filters.each { |v| puts "    #{v[:name]}(#{v[:args]}) #{v[:description]}" }
           end
         else
           puts "No resources found for '#{provider}'."

data/lib/outliers/collection.rb

@@ -17,11 +17,19 @@ module Outliers
        Outliers::Verifications::Shared.verifications + self.resource_class.verifications 
     end
 
+    def self.filters
+      []
+    end
+
     def self.resource_class
       array = self.to_s.gsub(/Collection$/, '').split('::')
       array.inject(Object) {|o,c| o.const_get c}
     end
 
+    def to_s
+      self.class.to_human
+    end
+
     def initialize(provider)
       @targets   = []
       @provider = provider
@@ -40,13 +48,39 @@ module Outliers
       @all = save
     end
 
+    def filter(args)
+      name  = args.keys.first
+      value = args.fetch name
+
+      logger.info "Applying filter '#{name}' with value '#{value}'."
+
+      unless self.public_methods.include? "filter_#{name}".to_sym
+        raise Exceptions::UnknownFilter.new "Unknown filter '#{name}'."
+      end
+
+      filtered_list = self.public_send "filter_#{name}", value
+
+      logger.warn "No resources match filter." unless filtered_list.any?
+
+      @all = filtered_list
+    end
+
     def verify(name, arguments={})
       name << "?" unless name =~ /^.*\?$/
 
-      logger.debug "Verifying resources '#{all_by_key.join(', ')}'."
+      unless all.any?
+        return { failing_resources: [], passing_resources: [] }
+      end
+
+      logger.info "Verifying '#{name}'."
+      logger.debug "Target resources '#{all_by_key.join(', ')}'."
+
+      unless verification_exists? name
+        raise Exceptions::UnknownVerification.new "Unkown verification '#{name}'."
+      end
 
       if collection_verification? name
-        send_verification self, name, arguments
+        send_collection_verification name, arguments
       else
         send_resources_verification name, arguments
       end
@@ -66,6 +100,13 @@ module Outliers
 
     private
 
+    def verification_exists?(name)
+      m = resource_class.instance_methods - resource_class.class.instance_methods
+      m += Outliers::Verifications::Shared.instance_methods
+      m -= [:source, :id, :method_missing]
+      m.include? name.to_sym
+    end
+
     def all_by_key
       all.map {|r| r.public_send key}
     end
@@ -82,17 +123,6 @@ module Outliers
       self.public_methods.include? name.to_sym
     end
 
-    def send_resources_verification(verification, arguments)
-      set_target_resources verification if targets.any?
-
-      results = map do |resource|
-        result = send_verification resource, verification, arguments
-        logger.debug "Verification of resource '#{resource.id}' #{result ? 'passed' : 'failed'}."
-        result
-      end
-      !results.include? false
-    end
-
     def set_target_resources(verification)
       logger.info "Verifying target '#{targets.join(', ')}'."
 
@@ -105,12 +135,27 @@ module Outliers
       @all
     end
 
+    def send_resources_verification(verification, arguments)
+      set_target_resources verification if targets.any?
+
+      failing_resources = reject do |resource|
+        result = send_verification resource, verification, arguments
+        logger.debug "Verification of resource '#{resource.id}' #{result ? 'passed' : 'failed'}."
+        result
+      end
+      { failing_resources: failing_resources, passing_resources: all - failing_resources }
+    end
+
+    def send_collection_verification(verification, arguments)
+      failing_resources = send_verification(self, verification, arguments)
+      { failing_resources: failing_resources, passing_resources: all - failing_resources }
+    end
+
     def send_verification(object, verification, arguments) 
       if object.method(verification).arity.zero?
         if arguments.any?
           raise Outliers::Exceptions::NoArgumentRequired.new "Verification '#{verification}' does not require an arguments."
         end
-
         object.public_send verification
       else
         if arguments.none?

data/lib/outliers/evaluation.rb

@@ -26,7 +26,7 @@ module Outliers
       targets_array = Array(targets)
 
       if targets_array.any?
-        logger.info "Verifying against '#{targets_array.join(', ')}' from '#{name}' collection."
+        logger.info "Verifying '#{targets_array.join(', ')}' from '#{name}' collection."
         collection.targets = targets_array
       end
       collection
@@ -36,14 +36,22 @@ module Outliers
       collection.exclude_by_key Array(exclusions)
     end
 
+    def filter(args)
+      collection.filter args.keys_to_s
+    end
+
     def verify(verification, arguments={})
-      collection.load_all
+      @resources_loaded ||= collection.load_all
 
-      r = collection.verify verification, arguments.keys_to_sym
+      verification_result = collection.verify verification, arguments.keys_to_sym
 
-      result = Outliers::Result.new :description => @name, :passed => r
+      result = Outliers::Result.new evaluation:        @name,
+                                    failing_resources: verification_result.fetch(:failing_resources),
+                                    passing_resources: verification_result.fetch(:passing_resources),
+                                    resource:          @collection,
+                                    verification:      verification
 
-      logger.info "Evaluation '#{result}'."
+      logger.info "Verification '#{verification}' #{result}."
 
       @run.results << result
     end

data/lib/outliers/exceptions.rb

@@ -12,6 +12,9 @@ module Outliers
     class ArgumentRequired < Base
     end
 
+    class InvalidBucket < Base
+    end
+
     class NoArgumentRequired < Base
     end
 
@@ -24,10 +27,10 @@ module Outliers
     class UnknownVerification < Base
     end
 
-    class UnknownProvider < Base
+    class UnknownFilter < Base
     end
 
-    class InvalidBucket < Base
+    class UnknownProvider < Base
     end
 
     class UnsupportedRegion < Base

data/lib/outliers/filters/aws/ec2/tags.rb

@@ -0,0 +1,38 @@
+module Outliers
+  module Filters
+    module Aws
+      module Ec2
+        module Tags
+
+          def filter_tag(value)
+            tag_name  = value.split(':').first
+            tag_value = value.split(':').last
+            logger.info "Filtering by tag '#{tag_name}' equals '#{tag_value}'."
+            all.select do |r|
+              if r.tags.has_key? tag_name
+                value = r.tags[tag_name]
+                result = value == tag_value
+                logger.debug "'#{r.id}' has tag with value '#{value}'. #{result ? 'Matches' : 'Does not match'} filter."
+                result
+              else
+                logger.debug "'#{r.id}' does not have tag '#{tag_name}'"
+                false
+              end
+            end
+          end
+
+          module_function
+
+          def self.filters
+            [
+              { name: 'tag',
+                description: 'Filter instances tagged with the given tag name and value.',
+                args: 'TAG_NAME:VALUE"' }
+            ]
+          end 
+
+        end
+      end
+    end
+  end
+end

data/lib/outliers/filters/aws/ec2.rb

@@ -0,0 +1 @@
+require 'outliers/filters/aws/ec2/tags'

data/lib/outliers/filters/aws.rb

@@ -0,0 +1 @@
+require 'outliers/filters/aws/ec2'

data/lib/outliers/filters.rb

@@ -0,0 +1 @@
+require 'outliers/filters/aws'

data/lib/outliers/providers.rb

@@ -1,5 +1,4 @@
 require 'outliers/providers/aws'
-require 'outliers/providers/github'
 
 module Outliers
   module Providers

data/lib/outliers/resource.rb

@@ -8,7 +8,7 @@ module Outliers
     end
 
     def self.verifications
-      {}
+      []
     end
 
     def initialize(source)

data/lib/outliers/resources/aws/ec2/image.rb

@@ -0,0 +1,13 @@
+module Outliers
+  module Resources
+    module Aws
+      module Ec2
+        class Image < Resource
+          def self.key
+            'image_id'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/outliers/resources/aws/ec2/image_collection.rb

@@ -0,0 +1,22 @@
+module Outliers
+  module Resources
+    module Aws
+      module Ec2
+        class ImageCollection < Collection
+
+          include Outliers::Filters::Aws::Ec2::Tags
+
+          def load_all
+            logger.debug "Loading private images owned by this account."
+            connect.images.with_owner(:self).map {|r| resource_class.new r}
+          end
+
+          def self.filters
+            Outliers::Filters::Aws::Ec2::Tags.filters
+          end 
+
+        end
+      end
+    end
+  end
+end

data/lib/outliers/resources/aws/ec2/instance_collection.rb

@@ -4,10 +4,16 @@ module Outliers
       module Ec2
         class InstanceCollection < Collection
 
+          include Outliers::Filters::Aws::Ec2::Tags
+
           def load_all
             connect.instances.map {|r| resource_class.new r}
           end
 
+          def self.filters
+            Outliers::Filters::Aws::Ec2::Tags.filters
+          end 
+
         end
       end
     end

data/lib/outliers/resources/aws.rb

@@ -4,6 +4,8 @@ require 'outliers/resources/aws/ec2/security_group'
 require 'outliers/resources/aws/ec2/security_group_collection'
 require 'outliers/resources/aws/ec2/instance'
 require 'outliers/resources/aws/ec2/instance_collection'
+require 'outliers/resources/aws/ec2/image'
+require 'outliers/resources/aws/ec2/image_collection'
 require 'outliers/resources/aws/elb/load_balancer'
 require 'outliers/resources/aws/elb/load_balancer_collection'
 require 'outliers/resources/aws/iam/user'

data/lib/outliers/resources.rb

@@ -1,5 +1,4 @@
 require 'outliers/resources/aws'
-require 'outliers/resources/github'
  
 module Outliers
   module Resources

data/lib/outliers/result.rb

@@ -1,11 +1,14 @@
 module Outliers
   class Result
 
-    attr_reader :description, :passed
+    attr_reader :evaluation, :failing_resources, :passing_resources, :resource, :verification
 
     def initialize(args)
-      @description = args[:description]
-      @passed      = args[:passed]
+      @evaluation        = args[:evaluation]
+      @failing_resources = args[:failing_resources]
+      @passing_resources = args[:passing_resources]
+      @resource          = args[:resource]
+      @verification      = args[:verification]
     end
 
     def to_s
@@ -13,11 +16,11 @@ module Outliers
     end
 
     def passed?
-      @passed == true
+      !failed?
     end
 
     def failed?
-      !passed?
+      @failing_resources.any?
     end
 
   end

data/lib/outliers/verifications/shared.rb

@@ -3,15 +3,13 @@ module Outliers
     module Shared
 
       def none_exist?
-        logger.debug 'Verifying no resources exist.'
-        logger.debug "Found #{all.empty? ? 'no resources' : all_by_key.join(',')}."
-        all.empty?
+        all
       end
 
       def equals?(args)
         list = Array(args[:keys])
-        logger.debug "Verifying '#{list.join(',')}' equals #{all.empty? ? 'no resources' : all_by_key.join(',')}."
-        list == all_by_key
+        logger.debug "Verifying '#{list.join(',')}' equals '#{all.empty? ? 'no resources' : all_by_key.join(',')}'."
+        all.reject {|r| list.include? r.id}
       end
 
       module_function

data/lib/outliers/version.rb

@@ -1,3 +1,3 @@
 module Outliers
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/lib/outliers.rb

@@ -5,6 +5,7 @@ require "outliers/collection"
 require "outliers/credentials"
 require "outliers/exceptions"
 require "outliers/evaluation"
+require "outliers/filters"
 require "outliers/provider"
 require "outliers/providers"
 require "outliers/resource"

data/outliers.gemspec

@@ -23,5 +23,4 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec", "~> 2.13.0"
 
   spec.add_runtime_dependency "aws-sdk", "1.14.1"
-  spec.add_runtime_dependency "github_api", "0.10.1"
 end

data/spec/collection_spec.rb

@@ -4,8 +4,6 @@ describe Outliers::Collection do
   let(:provider) { mock 'provider' }
   let(:resource1) { mock 'resource1' }
   let(:resource2) { mock 'resource2' }
-  let(:resource_class) { stub 'resource_class', key: 'name',
-                                                verifications_requiring_target: ['target_me?'] }
 
   subject { Outliers::Collection.new provider }
 
@@ -20,7 +18,7 @@ describe Outliers::Collection do
   context "#to_human" do
     it "should return the human name for this resource" do
       expect(Outliers::Resources::Aws::Ec2::SecurityGroupCollection.to_human).to eq('aws_ec2_security_group')
-      expect(Outliers::Resources::Github::RepoCollection.to_human).to eq('github_repo')
+      expect(Outliers::Resources::Aws::S3::BucketCollection.to_human).to eq('aws_s3_bucket')
     end
   end
 
@@ -37,6 +35,19 @@ describe Outliers::Collection do
     end
   end
 
+  context "#filter" do
+    it "should apply the given filter to resources" do
+      subject.should_receive('filter_tag').with('Name:test123').and_return [resource1]
+      subject.filter 'tag' => 'Name:test123'
+      expect(subject.all).to eq([resource1])
+    end
+
+    it "should raise an exception if the filter does not exist" do
+      expect { subject.filter('bogus' => 'Name:test123') }.
+        to raise_error Outliers::Exceptions::UnknownFilter
+    end
+  end
+
   context "#key" do
     it "should return the key for the resource" do
       expect(subject.key).to eq('name')
@@ -50,37 +61,52 @@ describe Outliers::Collection do
   end
 
   context "#verify" do
+    let(:resource_class) {
+      m = [:source, :id, :method_missing, :equals?, :none_exist?, :valid_resource?]
+      stub 'resource_class', key: 'name',
+                             verifications_requiring_target: ['target_me?'],
+                             instance_methods: m
+    }
     before do
       Outliers::Collection.stub :resource_class => resource_class
     end
 
     it "should verify the given verification against the colection" do
-      expect(subject.verify 'none?', {}).to be_false
+      expect(subject.verify 'none_exist?', {}).
+        to eq( { failing_resources: [resource1, resource2], passing_resources: [] } )
+    end
+
+    it "should raise unkown verification if the verification does not exist" do
+      expect { subject.verify 'none', {} }.to raise_error Outliers::Exceptions::UnknownVerification
     end
 
     it "should verify the given verification against the colection with options" do
-      expect(subject.verify 'equals?', :keys => ['resource1', 'resource2']).to be_true
+      expect(subject.verify 'equals?', :keys => ['resource1', 'resource2']).
+        to eq( { failing_resources: [], passing_resources: [resource1, resource2] } )
     end
 
     it "should verify the given verification against each resource in the collection" do
       [resource1, resource2].each {|r| r.define_singleton_method :valid_resource?, lambda { true } }
-      expect(subject.verify 'valid_resource?').to be_true
+      expect(subject.verify 'valid_resource?').
+        to eq( { failing_resources: [], passing_resources: [resource1, resource2] } )
     end
 
     it "should appaned a ? to the policy" do
-      expect(subject.verify 'none').to be_false
+      expect(subject.verify 'none_exist').
+        to eq( { failing_resources: [resource1, resource2], passing_resources: [] } )
     end
 
     it "should remove all but the target resources if one is required and given" do
       subject.targets = ['resource1', 'in-valid']
-      resource1.should_receive(:method).with('target_me?').and_return(stub 'method', :arity => 0)
-      resource1.should_receive(:target_me?)
-      expect(subject.verify 'target_me?').to be_true
+      resource1.should_receive(:method).with('valid_resource?').and_return(stub 'method', :arity => 0)
+      resource1.should_receive(:valid_resource?).and_return false
+      expect(subject.verify 'valid_resource?').
+        to eq( { failing_resources: [resource1], passing_resources: [] } )
     end
 
     it "should raise an error if the target resources does not exist" do
       subject.targets = ['in-valid']
-      expect { subject.verify 'target_me?' }.to raise_error(Outliers::Exceptions::TargetNotFound)
+      expect { subject.verify 'valid_resource?' }.to raise_error(Outliers::Exceptions::TargetNotFound)
     end
                  
     it "should raise an error if the verification requires arguments and none given" do
@@ -90,13 +116,20 @@ describe Outliers::Collection do
 
     it "should raise an error if the verification does not require arguments and arguments are given" do
       resource1.define_singleton_method :valid_resource?, lambda { true }
-      expect { subject.verify 'valid_resource?', 'unneeded argument' => 3 }.to raise_error(Outliers::Exceptions::NoArgumentRequired)
+      expect { subject.verify 'valid_resource?', 'unneeded argument' => 3 }.
+        to raise_error(Outliers::Exceptions::NoArgumentRequired)
+    end
+
+    it "should return empty passing and failing arrays if no resources exist in all" do
+      subject.stub :load_all => []
+      expect(subject.verify 'valid_resource?', {}).to eq( { failing_resources: [], passing_resources: [] } )
     end
 
     it "should verify the given verification against each resource in the collection with options" do
       resource1.should_receive(:valid_resource?).with('test_arg' => 2).and_return true
       resource2.should_receive(:valid_resource?).with('test_arg' => 2).and_return true
-      expect(subject.verify 'valid_resource?', 'test_arg' => 2).to be_true
+      expect(subject.verify 'valid_resource?', 'test_arg' => 2).
+        to eq( { failing_resources: [], passing_resources: [resource1, resource2] } )
     end
   end
 

data/spec/evaluation_spec.rb

@@ -2,7 +2,6 @@ require 'spec_helper'
 
 describe Outliers::Evaluation do
   let(:run) { mock 'run' }
-  let(:result) { mock 'result' }
   let(:connect) { subject.connect('test_credentials_1') }
   let(:resources) { subject.resources('security_group') }
   subject { Outliers::Evaluation.new :run => run, :name => 'test' }
@@ -73,21 +72,63 @@ describe Outliers::Evaluation do
       end
     end
 
+    context "#filter" do
+      it "should apply the given filter to the collection" do
+        resources.should_receive(:filter).with('tag' => 'Name:test123')
+        subject.filter 'tag' => 'Name:test123'
+      end
+
+      it "should convert keys in the args hash to strings" do
+        resources.should_receive(:filter).with('tag' => 'Name:test123')
+        subject.filter tag: 'Name:test123'
+      end
+    end
+
     context "#verify" do
+      let(:result1) { mock 'result1' }
+      let(:result2) { mock 'result2' }
+      let(:verification_response) { ( { passing_resources: ['1', '2'], failing_resources: ['3', '4'] } ) }
+
       before do
-        resources.should_receive(:load_all)
-        Outliers::Result.should_receive(:new).with(description: 'test', :passed => true).and_return result
-        run.should_receive(:results).and_return([])
+        resources.should_receive(:load_all).and_return ['resource1', 'resource2']
+        run.stub results: []
       end
 
       it "should verify the given method" do
-        resources.should_receive(:verify).with('test_verification?', {}).and_return(true)
-        expect(subject.verify('test_verification?', {})).to eq([result])
+        resources.should_receive(:verify).with('test_verification?', {}).and_return verification_response
+        Outliers::Result.should_receive(:new).with(evaluation:        'test',
+                                                   passing_resources: ['1','2'],
+                                                   failing_resources: ['3','4'],
+                                                   resource:          resources,
+                                                   verification:      'test_verification?').and_return result1
+        expect(subject.verify('test_verification?', {})).to eq([result1])
       end
 
       it "should convert all options to symbols" do
-        resources.should_receive(:verify).with('test_verification?', :test => false).and_return(true)
-        expect(subject.verify('test_verification?', { 'test' => false } )).to eq([result])
+        resources.should_receive(:verify).with('test_verification?', :test => false).and_return verification_response
+        Outliers::Result.should_receive(:new).with(evaluation:        'test',
+                                                   passing_resources: ['1','2'],
+                                                   failing_resources: ['3','4'],
+                                                   resource:          resources,
+                                                   verification:      'test_verification?').and_return result1
+        expect(subject.verify('test_verification?', { 'test' => false } )).to eq([result1])
+      end
+
+      it "should run verify multiple times in given evaluation" do
+        resources.should_receive(:verify).with('test_verification1?', :test => false).and_return verification_response
+        resources.should_receive(:verify).with('test_verification2?', :test => true).and_return verification_response
+        Outliers::Result.should_receive(:new).with(evaluation:        'test',
+                                                   passing_resources: ['1','2'],
+                                                   failing_resources: ['3','4'],
+                                                   resource:          resources,
+                                                   verification:      'test_verification1?').and_return result1
+        Outliers::Result.should_receive(:new).with(evaluation:        'test',
+                                                   passing_resources: ['1','2'],
+                                                   failing_resources: ['3','4'],
+                                                   resource:          resources,
+                                                   verification:      'test_verification2?').and_return result2
+        expect(subject.verify('test_verification1?', { 'test' => false })).to eq [result1]
+        expect(subject.verify('test_verification2?', { 'test' => true })).to eq [result1, result2]
       end
     end
 

data/spec/filters/aws/ec2/tags_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe Outliers::Filters::Aws::Ec2::Tags do
+  subject do
+    object = Object.new
+    object.extend Outliers::Filters::Aws::Ec2::Tags
+    object
+  end
+
+  let(:logger) { stub 'logger', debug: true, info: true }
+  let(:tags1) { mock 'tags1' }
+  let(:tags2) { mock 'tags2' }
+  let(:resource1) { stub 'resource1', tags: tags1, id: 'resource1' }
+  let(:resource2) { stub 'resource2', tags: tags2, id: 'resource2' }
+
+  before do
+    subject.stub :logger => logger
+    subject.stub :all => [resource1, resource2]
+  end
+
+  it "should return the list of instances filtered by the given tag name and value" do
+    tags1.should_receive(:has_key?).with('Name').and_return true
+    tags2.should_receive(:has_key?).with('Name').and_return false
+    tags1.should_receive(:[]).with('Name').and_return 'test123'
+    expect(subject.filter_tag('Name:test123')).to eq([resource1])
+  end
+end

data/spec/provider_spec.rb

@@ -4,25 +4,25 @@ describe Outliers::Provider do
   subject { Outliers::Provider }
 
   context "#connect_to" do
-    let(:credentials) { ( { :name      => "test_credentials_1",
-                            "provider" => "github",
-                            "token"    => "abc" } ) }
+    let(:credentials) { ( { :name               => "test_credentials_1",
+                            "provider"          => "aws_ec2",
+                            "secret_access_key" => "abc",
+                            "access_key_id"     => "123" } ) }
 
     it "should connect to the provider specified in the given credentials" do
-      expect(subject.connect_to(credentials).class).to eq(Outliers::Providers::Github)
+      expect(subject.connect_to(credentials).class).to eq(Outliers::Providers::Aws::Ec2)
     end
 
     it "should set the credentials instance variable" do
       expect(subject.connect_to(credentials).credentials).
-        to eq({ :name => "test_credentials_1", "provider" => "github", "token" => "abc" })
+        to eq({ :name               => "test_credentials_1",
+                "provider"          => "aws_ec2",
+                "secret_access_key" => "abc",
+                "access_key_id"     => "123" })
     end
   end
 
   context "#to_human" do
-    it "should return the name a human would use to access the provider" do
-      expect(Outliers::Providers::Github.to_human).to eq('github')
-    end
-
     it "should return the name a human would use to access the provider" do
       expect(Outliers::Providers::Aws::Rds.to_human).to eq('aws_rds')
     end

data/spec/results_spec.rb

@@ -2,7 +2,12 @@ require 'spec_helper'
 
 describe Outliers::Result do
   context "passing" do
-    subject { Outliers::Result.new description: 'stuff', passed: true }
+    subject { Outliers::Result.new evaluation:        'evalme',
+                                   failing_resources: [],
+                                   passing_resources: ['key1', 'key2'],
+                                   resource:          'instance',
+                                   verification:      'vpc' }
+                                   
     it "should return passed" do
       expect(subject.to_s).to eq 'passed'
     end
@@ -11,18 +16,26 @@ describe Outliers::Result do
       expect(subject.passed?).to be_true
     end
 
-    it "should return false for passing verification" do
+    it "should return false for failing verification" do
       expect(subject.failed?).to be_false
     end
+
+    it "should return the result information" do
+      expect(subject.passing_resources).to eq(['key1', 'key2'])
+    end
   end
 
   context "failing" do
-    subject { Outliers::Result.new description: 'stuff', passed: false }
-    it "should return passed" do
+    subject { Outliers::Result.new evaluation:        'evalme',
+                                   failing_resources: ['key3', 'key4'],
+                                   passing_resources: [],
+                                   resource:          'instance',
+                                   verification:      'vpc' }
+    it "should return failed" do
       expect(subject.to_s).to eq 'failed'
     end
 
-    it "should return false for failing verification" do
+    it "should return false for passing verification" do
       expect(subject.passed?).to be_false
     end
 

data/spec/run_spec.rb

@@ -47,8 +47,9 @@ describe Outliers::Run do
   end
 
   context "returning results" do
-    let(:result1) { Outliers::Result.new description: 'result1', passed: true }
-    let(:result2) { Outliers::Result.new description: 'result2', passed: false }
+    let(:result1) { Outliers::Result.new name: 'result1', passing_resources: [], failing_resources: [], evaluation: 'test', verification: 'ver' }
+    let(:result2) { Outliers::Result.new name: 'result2', passing_resources: [], failing_resources: ['failed'], evaluation: 'test', verification: 'ver' }
+
     before do
       subject.results << result1
       subject.results << result2

data/spec/verifications/shared_spec.rb

@@ -2,6 +2,8 @@ require 'spec_helper'
 
 describe Outliers::Verifications::Shared do
   subject { Object.new.extend Outliers::Verifications::Shared }
+  let(:resource1) { stub "resource1", id: 'resource1' }
+  let(:resource2) { stub "resource2", id: 'resource2' }
 
   before do
     logger_stub = stub 'logger', :debug => true
@@ -11,25 +13,30 @@ describe Outliers::Verifications::Shared do
   context "#none_exist?" do
     it "should be true if no resources returned" do
       subject.stub :all => []
-      expect(subject.none_exist?).to be_true
+      expect(subject.none_exist?).to eq([])
     end
 
     it "should be false if  resources returned" do
-      subject.stub :all_by_key => ['test']
-      subject.stub :all => ['test']
-      expect(subject.none_exist?).to be_false
+      subject.stub :all_by_key => ['resource1']
+      subject.stub :all => ['resource1']
+      expect(subject.none_exist?).to eq(['resource1'])
     end
   end
 
   context "#equals?" do
-    it "should verify the list of resources equals the list of keys" do
-      subject.stub :all_by_key => ['test'], :all => ['test_resource']
-      expect(subject.equals?(:keys => ['test'])).to be_true
+    it "should verify the list of resources equals the list of keys and return no failing reosurces" do
+      subject.stub :all_by_key => ['resource1'], :all => [resource1]
+      expect(subject.equals?(:keys => ['resource1'])).to eq([])
     end
 
-    it "should verify the list of resources equals the single key" do
-      subject.stub :all_by_key => ['test'], :all => ['test_resource']
-      expect(subject.equals?(:keys => 'test')).to be_true
+    it "should verify the list of resources equals the single key and return no failing resources" do
+      subject.stub :all_by_key => ['resource1'], :all => [resource1]
+      expect(subject.equals?(:keys => 'resource1')).to eq([])
+    end
+
+    it "should return resources which do not match the given list" do
+      subject.stub :all_by_key => ['resource1', 'resource2'], :all => [resource1, resource2]
+      expect(subject.equals?(:keys => 'resource1')).to eq([resource2])
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: outliers
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Brett Weaver
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-13 00:00:00.000000000 Z
+date: 2013-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,20 +66,6 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.14.1
-- !ruby/object:Gem::Dependency
-  name: github_api
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.10.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.10.1
 description: Configuraiton verification framework.
 email:
 - brett@weav.net
@@ -108,6 +94,10 @@ files:
 - lib/outliers/credentials.rb
 - lib/outliers/evaluation.rb
 - lib/outliers/exceptions.rb
+- lib/outliers/filters.rb
+- lib/outliers/filters/aws.rb
+- lib/outliers/filters/aws/ec2.rb
+- lib/outliers/filters/aws/ec2/tags.rb
 - lib/outliers/mixins.rb
 - lib/outliers/provider.rb
 - lib/outliers/providers.rb
@@ -120,12 +110,13 @@ files:
 - lib/outliers/providers/aws/rds.rb
 - lib/outliers/providers/aws/s3.rb
 - lib/outliers/providers/aws/sqs.rb
-- lib/outliers/providers/github.rb
 - lib/outliers/resource.rb
 - lib/outliers/resources.rb
 - lib/outliers/resources/aws.rb
 - lib/outliers/resources/aws/cloud_formation/stack.rb
 - lib/outliers/resources/aws/cloud_formation/stack_collection.rb
+- lib/outliers/resources/aws/ec2/image.rb
+- lib/outliers/resources/aws/ec2/image_collection.rb
 - lib/outliers/resources/aws/ec2/instance.rb
 - lib/outliers/resources/aws/ec2/instance_collection.rb
 - lib/outliers/resources/aws/ec2/security_group.rb
@@ -142,9 +133,6 @@ files:
 - lib/outliers/resources/aws/s3/bucket_collection.rb
 - lib/outliers/resources/aws/sqs/queue.rb
 - lib/outliers/resources/aws/sqs/queue_collection.rb
-- lib/outliers/resources/github.rb
-- lib/outliers/resources/github/repo.rb
-- lib/outliers/resources/github/repo_collection.rb
 - lib/outliers/result.rb
 - lib/outliers/run.rb
 - lib/outliers/verifications.rb
@@ -154,6 +142,7 @@ files:
 - spec/collection_spec.rb
 - spec/credentials_spec.rb
 - spec/evaluation_spec.rb
+- spec/filters/aws/ec2/tags_spec.rb
 - spec/fixtures/credentials1.yml
 - spec/fixtures/credentials2.yml
 - spec/helpers/fixtures.rb
@@ -194,6 +183,7 @@ test_files:
 - spec/collection_spec.rb
 - spec/credentials_spec.rb
 - spec/evaluation_spec.rb
+- spec/filters/aws/ec2/tags_spec.rb
 - spec/fixtures/credentials1.yml
 - spec/fixtures/credentials2.yml
 - spec/helpers/fixtures.rb

data/lib/outliers/providers/github.rb

@@ -1,23 +0,0 @@
-require "github_api"
-
-module Outliers
-  module Providers
-    class Github < Provider
-
-      def settings(args)
-        @token = args[:token]
-      end
-
-      def connect
-        c = ::Github.new 
-        c.oauth_token = @token if @token
-        c
-      end
-
-      def self.credential_arguments
-        { 'token' => 'Github API token.' }
-      end
-
-    end
-  end
-end

data/lib/outliers/resources/github/repo.rb

@@ -1,24 +0,0 @@
-module Outliers
-  module Resources
-    module Github
-      class Repo < Resource
-        def self.verifications
-          [
-            { name: 'private',
-              description: 'Repo is private.' },
-            { name: 'public',
-              description: 'Repo is public.' }
-          ]
-        end
-
-        def private?
-          source.private
-        end
-
-        def public?
-          !source.private
-        end
-      end
-    end
-  end
-end

data/lib/outliers/resources/github/repo_collection.rb

@@ -1,13 +0,0 @@
-module Outliers
-  module Resources
-    module Github
-      class RepoCollection < Collection
-
-        def load_all
-          connect.repos.list(:type => 'all').map {|r| resource_class.new r}
-        end
-
-      end
-    end
-  end
-end

data/lib/outliers/resources/github.rb

@@ -1,2 +0,0 @@
-require 'outliers/resources/github/repo'
-require 'outliers/resources/github/repo_collection'