outliers 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca2a95007c8c6b9d85847b7c857e8e5f54e72859
-  data.tar.gz: 273c267dc763814b17162cf394cc453e77ba2253
+  metadata.gz: 74e9cb409e6eef3f41b71f7546b03a331acd8e78
+  data.tar.gz: 3bf117be6746ca6f358e53b063c8c46226ad8d23
 SHA512:
-  metadata.gz: efbb5c48979eafc1001695502ad892ae799517497b789e6b56d034b44e0e3cdf74fa40c950cf77e9fd329cd0afbe1fb4ab612497c6b0a33ab7d00e95c1b75cba
-  data.tar.gz: a2a0da57894add6de86c37a749d172a8174476b12f1d8665ebe68df9b11c31271bab95de1c1171f401cc4f8a2619e9726ad426f62b66914ec0e1b280f8aa74e4
+  metadata.gz: 64869c01820ae1d1e323c9531bafd25c10d57ef2660b4b6f505967a81256b974e05d4f9e9732bbb31dc785c7116227651a65fa2100e2330b8715e2452562d4c9
+  data.tar.gz: c9e49150e28457a4152b9cb5572ab9851e34c3fcf9742782d50a12afbc0043cacbb2b740addcc740256991486a4efeca4bb5ec721ae828c445d952665b5e00e6

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## HEAD
+
+## 0.3.0
+
+* Remove CLI support
+* Pull resource and provider info from reference.yaml
+* Support running multiple evaluation threads
+
 ## 0.2.0
 
 * Added filters

data/README.md

@@ -9,8 +9,8 @@ Outliers is a framework for verifying configuration of resources.
 * Applications and teams rely on multiple service providers (AWS, etc).
 * Providers deliver like resources with complex configuration (EC2 Instances, S3 Buckets, etc).
 * Resource configuration can be verified (launched from given AMI, contain private objects, etc).
-* The resources can be included or excluded by their ID (Instance ID, Object Key, etc).
-* Resources can be included in the list by matching a filter (Instance has tag 'x' with value 'y').
+* Resources can be targeted or excluded by their ID (Instance ID, Object Key, etc).
+* Resources can be targeted or excluded by matching a filter (Instance has tag 'x' with value 'y').
 * Those not passing verifications, are flagged as Outliers.
 
 ## Requirements
@@ -47,53 +47,9 @@ Multiple accounts can be specified, to add a prod and preprod AWS account:
 
 Depending on the provider, different keys and values are required.
 
-For a list of providers:
-
-    outliers providers
-
 ## Usage
 
-Outliers can be used in two modes, as a **CLI** or a **DSL**.
-
-The CLI is good for testing and quick verifications.
-
-The DSL can be used to build up comprehensive list of verifications for a project or company.
-
-### CLI
-
-To verify all EC2 instances are in a VPC:
-
-    outliers evaluate -c aws_prod -p aws_ec2 -r instance -v vpc
-
-Credential keys can be overriden. To specify region us-west-1.
-
-    outliers evaluate -c aws_prod -p aws_ec2 -r instance -v vpc -c region=us-west-1
-
-Verifications may require arguments. To verify your RDS databases have a 2 day retention period:
-
-    outliers evaluate -c aws_prod -p aws_rds -r db_instance -v backup_retention_period -a days=2
-
-Verifications may accept multiple values for an argument. Values are separated by commas.
-
-To verify EC2 instances are launched from a list of known images:
-
-    outliers evaluate -c aws_prod -p aws_ec2 -r instance -v valid_image_id -a image_ids=ami-12345678,ami-87654321
-
-To only target a specific resource:
-
-    outliers evaluate -c aws_prod -p aws_ec2 -r instance -t i-87654321
-
-To exclude resources that are known exceptions:
-
-    outliers evaluate -c aws_prod -p aws_ec2 -r instance -e i-12345678
-
-Resources have attributes which can be used to filter target resources. To filter instances who have tag 'Name' equal to 'web'.
-
-    outliers evaluate -c aws_prod -p aws_ec2 -r instance -f 'tag=Name:web'
-
-### DSL
-
-To run Outliers as a DSL
+Outlier's DSL can be used to build up comprehensive list of verifications for a project or application.
 
 * Create a directory to store your evaluations. 
 * Evalutions are read from from files within the directory.
@@ -195,7 +151,9 @@ Sometimes you want to exclude resources that are known exceptions, to exclude an
       verify 'valid_image_id', image_ids: ['ami-12345678','ami-87654321']
     end
 
-Resources have attributes which can be used to filter target resources. To filter instances who have tag 'Name' equal to 'web'.
+Resources have attributes which can be used to filter target resources.
+
+To filter instances who have tag 'Name' equal to 'web'.
 
     evaluate do
       connect 'aws_prod', provider: 'aws_ec2', region: 'us-west-1'
@@ -204,24 +162,6 @@ Resources have attributes which can be used to filter target resources. To filte
       verify 'valid_image_id', image_ids: ['ami-12345678','ami-87654321']
     end
 
-### Help
-
-For a list of providers and required credentials:
-
-    outliers providers
-
-For a list of resources, and available verifications, for a given provider:
-
-    outliers resources -p PROVIDER_NAME
-
-For a fule list of commands run:
-
-    outliers -h
-
-For full help on a command, append -h:
-
-    outliers evaluate -h
-
 ## Contributing
 
 1. Fork it

data/lib/outliers/cli/process.rb

@@ -2,7 +2,7 @@ module Outliers
   module CLI
     class Process
       def process
-        @options = {}
+        @options = { threads: 1 }
 
         option_parser.parse!
 
@@ -11,9 +11,14 @@ module Outliers
         @logger = Outliers.logger
         @run    = Run.new
 
+        if @options[:threads] > 1
+          @run.threaded     = true
+          @run.thread_count = @options[:threads]
+        end
+
         begin
           @run.credentials = Credentials.load_from_file "#{ENV['HOME']}/.outliers.yml"
-          @run.process_evaluations_in_config_folder
+          @run.process_evaluations_in_dir
         rescue Outliers::Exceptions::Base => e
           @logger.error e.message
           exit 1
@@ -26,7 +31,7 @@ module Outliers
 
         @run.failed.each do |f|
           @logger.info "Evaluation '#{f.evaluation}' verification '#{f.verification}' of '#{f.resource}' failed."
-          @logger.debug "Failing resource IDs '#{f.failing_resources.map{|r| r.id}.join(', ')}'"
+          @logger.info "Failing resource IDs '#{f.failing_resources.map{|r| r.id}.join(', ')}'"
         end
 
         @logger.info "(#{failed} evaluations failed, #{passed} evaluations passed.)"
@@ -48,6 +53,10 @@ module Outliers
         OptionParser.new do |opts|
           opts.banner = "Usage: outliers process [options]"
 
+          opts.on("-t", "--threads [THREADS]", "Maximum number of evaluations threads to run concurrently (Default: 1).") do |o|
+            @options[:threads] = o.to_i
+          end
+
           opts.on("-d", "--directory [DIRECTORY]", "Directory containing evaluations to load.") do |o|
             @options[:directory] = o
           end

data/lib/outliers/cli.rb

@@ -1,9 +1,6 @@
 require 'optparse'
 
-require 'outliers/cli/evaluate'
 require 'outliers/cli/process'
-require 'outliers/cli/providers'
-require 'outliers/cli/resources'
 
 module Outliers
   module CLI
@@ -12,20 +9,6 @@ module Outliers
       cmd = ARGV.shift
 
       case cmd
-      when 'evaluate'
-        begin
-          CLI::Evaluate.new.evaluate
-        rescue OptionParser::MissingArgument => e
-          puts e.message
-          exit 1
-        end
-      when 'providers'
-        begin
-          CLI::Providers.new.providers
-        rescue OptionParser::MissingArgument => e
-          puts e.message
-          exit 1
-        end
       when 'process'
         begin
           CLI::Process.new.process
@@ -33,13 +16,6 @@ module Outliers
           puts e.message
           exit 1
         end
-      when 'resources'
-        begin
-          CLI::Resources.new.resources
-        rescue OptionParser::MissingArgument => e
-          puts e.message
-          exit 1
-        end
       when '-v'
         puts OUTLIERS::VERSION
       else

data/lib/outliers/collection.rb

@@ -37,15 +37,15 @@ module Outliers
     end
 
     def each &block  
-      all.each do |resource|
+      list.each do |resource|
         block.call resource
       end  
     end
 
     def exclude_by_key(exclusions)
       @logger.info "Excluding the following resources: '#{exclusions.join(',')}'."
-      save = all.reject {|u| exclusions.include? u.public_send key}
-      @all = save
+      save = list.reject {|u| exclusions.include? u.public_send key}
+      @list = save
     end
 
     def filter(args)
@@ -62,18 +62,18 @@ module Outliers
 
       logger.warn "No resources match filter." unless filtered_list.any?
 
-      @all = filtered_list
+      @list = filtered_list
     end
 
     def verify(name, arguments={})
       name << "?" unless name =~ /^.*\?$/
 
-      unless all.any?
+      unless list.any?
         return { failing_resources: [], passing_resources: [] }
       end
 
       logger.info "Verifying '#{name}'."
-      logger.debug "Target resources '#{all_by_key.join(', ')}'."
+      logger.debug "Target resources '#{list_by_key.join(', ')}'."
 
       unless verification_exists? name
         raise Exceptions::UnknownVerification.new "Unkown verification '#{name}'."
@@ -86,8 +86,8 @@ module Outliers
       end
     end
 
-    def all
-      @all ||= load_all
+    def list
+      @list ||= load_all
     end
 
     def key
@@ -107,8 +107,8 @@ module Outliers
       m.include? name.to_sym
     end
 
-    def all_by_key
-      all.map {|r| r.public_send key}
+    def list_by_key
+      list.map {|r| r.public_send key}
     end
 
     def connect
@@ -126,13 +126,13 @@ module Outliers
     def set_target_resources(verification)
       logger.info "Verifying target '#{targets.join(', ')}'."
 
-      @all = all.select {|r| targets.include? r.id }
+      @list = list.select {|r| targets.include? r.id }
 
-      unless all.any?
+      unless list.any?
         raise Outliers::Exceptions::TargetNotFound.new "No resources found matching one or more of '#{targets}'."
       end
 
-      @all
+      @list
     end
 
     def send_resources_verification(verification, arguments)
@@ -143,12 +143,12 @@ module Outliers
         logger.debug "Verification of resource '#{resource.id}' #{result ? 'passed' : 'failed'}."
         result
       end
-      { failing_resources: failing_resources, passing_resources: all - failing_resources }
+      { failing_resources: failing_resources, passing_resources: list - failing_resources }
     end
 
     def send_collection_verification(verification, arguments)
       failing_resources = send_verification(self, verification, arguments)
-      { failing_resources: failing_resources, passing_resources: all - failing_resources }
+      { failing_resources: failing_resources, passing_resources: list - failing_resources }
     end
 
     def send_verification(object, verification, arguments) 

data/lib/outliers/filters/aws/ec2/tags.rb

@@ -8,7 +8,7 @@ module Outliers
             tag_name  = value.split(':').first
             tag_value = value.split(':').last
             logger.info "Filtering by tag '#{tag_name}' equals '#{tag_value}'."
-            all.select do |r|
+            list.select do |r|
               if r.tags.has_key? tag_name
                 value = r.tags[tag_name]
                 result = value == tag_value

data/lib/outliers/info.rb

@@ -0,0 +1,12 @@
+require 'yaml'
+
+module Outliers
+  module Info
+    module_function
+
+    def reference
+      YAML.load_file(File.expand_path(File.join(File.dirname(__FILE__), '../../reference.yaml')))
+    end
+
+  end
+end

data/lib/outliers/provider.rb

@@ -14,10 +14,6 @@ module Outliers
       (self.to_s.split('::') - ['Outliers', 'Providers']).map { |p| p.underscore }.join('_').downcase
     end
 
-    def self.credential_arguments
-      {}
-    end
-
     def initialize(credentials)
       @credentials = credentials
       @logger      = Outliers.logger

data/lib/outliers/providers/aws/cloud_formation.rb

@@ -5,12 +5,8 @@ module Outliers
 
         include Shared
 
-        def self.credential_arguments
-          Shared.credential_arguments
-        end
-
         def connect
-          logger.info "Connecting to region '#{@region}'." unless @cf
+          logger.debug "Connecting to region '#{@region}'." unless @cf
           @cf ||= ::AWS::CloudFormation.new config
         end
 

data/lib/outliers/providers/aws/ec2.rb

@@ -5,12 +5,8 @@ module Outliers
 
         include Shared
 
-        def self.credential_arguments
-          Shared.credential_arguments
-        end
-
         def connect
-          logger.info "Connecting to region '#{@region}'." unless @ec2
+          logger.debug "Connecting to region '#{@region}'." unless @ec2
           @ec2 ||= ::AWS::EC2.new config
         end
 

data/lib/outliers/providers/aws/elb.rb

@@ -5,12 +5,8 @@ module Outliers
 
         include Shared
 
-        def self.credential_arguments
-          Shared.credential_arguments
-        end
-
         def connect
-          logger.info "Connecting to region '#{@region}'." unless @elb
+          logger.debug "Connecting to region '#{@region}'." unless @elb
           @elb ||= ::AWS::ELB.new config
         end
 

data/lib/outliers/providers/aws/iam.rb

@@ -5,12 +5,8 @@ module Outliers
 
         include Shared
 
-        def self.credential_arguments
-          Shared.credential_arguments
-        end
-
         def connect
-          logger.info "Connecting to region '#{@region}'." unless @iam
+          logger.debug "Connecting to region '#{@region}'." unless @iam
           @iam ||= ::AWS::IAM.new config
         end
 

data/lib/outliers/providers/aws/rds.rb

@@ -5,12 +5,8 @@ module Outliers
 
         include Shared
 
-        def self.credential_arguments
-          Shared.credential_arguments
-        end
-
         def connect
-          logger.info "Connecting to region '#{@region}'." unless @rds
+          logger.debug "Connecting to region '#{@region}'." unless @rds
           @rds ||= ::AWS::RDS.new config
         end
 

data/lib/outliers/providers/aws/s3.rb

@@ -5,12 +5,8 @@ module Outliers
 
         include Shared
 
-        def self.credential_arguments
-          Shared.credential_arguments
-        end
-
         def connect
-          logger.info "Connecting to region '#{@region}'." unless @iam
+          logger.debug "Connecting to region '#{@region}'." unless @iam
           @iam ||= ::AWS::S3.new config
         end
 

data/lib/outliers/providers/aws/{base.rb → shared.rb}

@@ -17,16 +17,6 @@ module Outliers
             :region            => @region }
         end
 
-        module_function
-
-        def credential_arguments
-          {
-            'access_key_id'     => 'AWS Account Access Key',
-            'secret_access_key' => 'AWS Account Secret Key',
-            'region'            => 'AWS Region (Default us-east-1)'
-          }
-        end
-
       end
     end
   end

data/lib/outliers/providers/aws/sqs.rb

@@ -5,12 +5,8 @@ module Outliers
 
         include Shared
 
-        def self.credential_arguments
-          Shared.credential_arguments
-        end
-
         def connect
-          logger.info "Connecting to region '#{@region}'." unless @sqs
+          logger.debug "Connecting to region '#{@region}'." unless @sqs
           @sqs ||= ::AWS::SQS.new config
         end
 

data/lib/outliers/providers/aws.rb

@@ -1,4 +1,4 @@
-require 'outliers/providers/aws/base'
+require 'outliers/providers/aws/shared'
 
 require 'outliers/providers/aws/cloud_formation'
 require 'outliers/providers/aws/ec2'

data/lib/outliers/resources/aws/ec2/instance.rb

@@ -7,22 +7,6 @@ module Outliers
             'instance_id'
           end
 
-          def self.verifications
-            [
-              { name: 'classic',
-                description: 'Instance is in AWS Classic (No VPC).' },
-              { name: 'source_dest_check',
-                description: 'Instance source dest check set to true.' },
-              { name: 'running',
-                description: 'Instance status is running.' },
-              { name: 'valid_image_id',
-                description: 'ami_ids=ami_id1,ami_id2 - Instances Image ID (AMI) is in given list.',
-                args: 'image_ids: [IMAGE_ID1, IMAGEID2]' },
-              { name: 'vpc',
-                description: 'Instance is in a VPC.' }
-            ]
-          end
-
           def classic?
             !vpc?
           end

data/lib/outliers/resources/aws/ec2/security_group.rb

@@ -3,13 +3,6 @@ module Outliers
     module Aws
       module Ec2
         class SecurityGroup < Resource
-          def self.verifications
-            [
-              { name: 'no_public_internet_ingress',
-                description: 'Security Group has no rules open to "0.0.0.0/0".' }
-            ]
-          end
-
           def no_public_internet_ingress?
             logger.debug "Verifying '#{id}'."
             source.ip_permissions.select do |i|

data/lib/outliers/resources/aws/elb/load_balancer.rb

@@ -3,13 +3,6 @@ module Outliers
     module Aws
       module Elb
         class LoadBalancer < Resource
-          def self.verifications
-            [
-              { name: 'ssl_certificates_valid',
-                description: 'Validates all SSL certificates associated with an ELB are valid for given number of days',
-                args: 'days: DAYS' }
-            ]
-          end
 
           def ssl_certificates_valid?(args)
             days = args[:days]

data/lib/outliers/resources/aws/iam/user.rb

@@ -3,16 +3,6 @@ module Outliers
     module Aws
       module Iam
         class User < Resource
-          def self.verifications
-            [
-              { name: 'mfa_enabled',
-                description: 'Verify MFA enabled for user.' },
-              { name: 'no_access_keys',
-                description: 'Verify user has no access keys.' },
-              { name: 'no_password_set',
-                description: 'Verify password not set for user.' }
-            ]
-          end
 
           def no_access_keys?
             logger.debug "#{id} has #{access_keys.count} access key(s)."

data/lib/outliers/resources/aws/rds/db_instance.rb

@@ -7,16 +7,6 @@ module Outliers
             'db_instance_identifier'
           end
 
-          def self.verifications
-            [ 
-              { name: 'backup_retention_period',
-                description: 'Validate the backup retention period equals given days for the db_instance.',
-                args: 'days: DAYS' },
-              { name: 'multi_az',
-                description: 'RDS Multi AZ set to yes.' }
-            ]
-          end
-
           def backup_retention_period?(args)
             days = args[:days]
 

data/lib/outliers/resources/aws/s3/bucket.rb

@@ -4,19 +4,6 @@ module Outliers
       module S3
         class Bucket < Resource
 
-          def self.verifications
-            [
-              { name: 'empty',
-                description: 'Bucket has no objects.' },
-              { name: 'no_public_objects',
-                description: 'Bucket has no public accessible objects.' },
-              { name: 'configured_as_website',
-                description: 'Bucket is configured as a website.' },
-              { name: 'not_configured_as_website',
-                description: 'Bucket is not configured as a website.' }
-            ]
-          end
-
           def empty?
             logger.debug "Bucket #{id} has #{count} objects."
 

data/lib/outliers/resources.rb

@@ -4,6 +4,10 @@ module Outliers
   module Resources
     module_function
 
+    def list
+      Outliers::Resources.collections
+    end
+
     def collections
       all_the_modules.select {|m| (m.is_a? Class) && (m.to_s =~ /Collection$/)}
     end

data/lib/outliers/run.rb

@@ -1,24 +1,35 @@
 module Outliers
   class Run
-    attr_accessor :credentials, :results
-
-    def initialize
-      @results = []
+    attr_accessor :credentials, :results, :threads, :threaded, :thread_count
+
+    def initialize(options={})
+      @results                  = []
+      @threads                  = []
+      @threaded                 = false
+      @thread_count             = 1
+      Thread.abort_on_exception = true
     end
 
-    def process_evaluations_in_config_folder
-      evaluations_path = File.join Outliers.config_path
-      files = Dir.glob(File.join(evaluations_path, '**', '*'))
+    def process_evaluations_in_dir
       files.each do |file|
         next if File.directory? file
         next if File.extname(file) != '.rb'
         logger.info "Processing '#{file}'."
         self.instance_eval File.read(file)
       end
+
+      threads.each {|t| t.join}
     end
 
     def evaluate(name='unspecified', &block)
-      Evaluation.new(:name => name, :run => self).instance_eval &block
+      while Thread.list.count > thread_count
+        logger.info "Maximum concurrent threads running, sleeping."
+        sleep 2
+      end
+
+      evaluation = Proc.new { Evaluation.new(:name => name, :run => self).instance_eval &block }
+
+      threaded ? threads << Thread.new { evaluation.call } : evaluation.call
     end
 
     def passed
@@ -31,6 +42,11 @@ module Outliers
 
     private
 
+    def files
+      evaluations_path = File.join Outliers.config_path
+      files = Dir.glob File.join(evaluations_path, '**', '*')
+    end
+
     def logger
       @logger ||= Outliers.logger
     end