cfn-nag 0.4.82 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da96aa37c7198a301552e939d2013246b2fe285c36719cb5e3cb44cc5e757ddf
-  data.tar.gz: 4cdca4308db2101e4f7ae1e2403eb4944715912b6403b1e5e21629d392148e5e
+  metadata.gz: c3d07f5b4eb55109314e6ce4d6361da04baa6d194e5ff4ec7eb8b39f55a345da
+  data.tar.gz: 3761581983506be2911e8cdaf504590f68420f09ae9b368789db83979a161bbb
 SHA512:
-  metadata.gz: 38e89b36905cb8573192787400cec3f43255637f1a9285bd4a72e63f55bc978bacdff0c59bbb6ce4d0f5fd222319e95076e1fe95153dcfc23527a8a8e6edaf56
-  data.tar.gz: 621c466546551f7157be0a0008205d558f3c61094b066b5375b5186829ef59ce964e61190134217dceaa0a46c810554a60bab935fdfe4edc218c96cdb07df43e
+  metadata.gz: c678f4fc50b3c4419aab7d66956212d573413be0745469d3bf9325241d2f04b0a43b76013f830806863a7ac3b0683e5c54aa352dd2b66fe93cd64bf8fdb2ad2e
+  data.tar.gz: 00ff6c2bf813e325897ec761a365e334f9d8267892eccf82d1d0118261ab42226c7cde3e6857ff082b3cfab1ace3e3b0fd80f5d087b40718b5b13bddb8f92be2

data/bin/cfn_nag_rules

@@ -1,11 +1,11 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require 'trollop'
+require 'optimist'
 require 'cfn-nag'
 require 'rubygems/specification'
 
-opts = Trollop.options do
+opts = Optimist.options do
   version Gem::Specification.find_by_name('cfn-nag').version
 
   opt :rule_directory, 'Extra rule directories', type: :io,
@@ -18,6 +18,11 @@ opts = Trollop.options do
       'Format of results: [csv, json, txt]',
       type: :string,
       default: 'txt'
+
+  opt :rule_repository,
+      'Path(s)s to rule repository to include in rule discovery',
+      type: :strings,
+      required: false
 end
 
 profile_definition = nil
@@ -25,8 +30,14 @@ unless opts[:profile_path].nil?
   profile_definition = IO.read(opts[:profile_path])
 end
 
+rule_repository_definitions = []
+opts[:rule_repository]&.each do |rule_repository|
+  rule_repository_definitions << IO.read(rule_repository)
+end
+
 rule_dumper = CfnNagRuleDumper.new(profile_definition: profile_definition,
                                    rule_directory: opts[:rule_directory],
-                                   output_format: opts[:output_format])
+                                   output_format: opts[:output_format],
+                                   rule_repository_definitions: rule_repository_definitions)
 
 rule_dumper.dump_rules

data/lib/cfn-nag/blacklist_loader.rb

@@ -38,6 +38,6 @@ class BlackListLoader
   def check_valid_rule_id(rule_id)
     return true unless @rules_registry.by_id(rule_id).nil?
 
-    raise "#{rule_id} is not a legal rule identifier from: #{rules_ids}"
+    raise "#{rule_id} is not a legal rule identifier from: #{@rules_registry.ids}"
   end
 end

data/lib/cfn-nag/cfn_nag.rb

@@ -87,12 +87,15 @@ class CfnNag
                                       parameter_values_string,
                                       true,
                                       condition_values_string
-      violations += @config.custom_rule_loader.execute_custom_rules(cfn_model)
+      violations += @config.custom_rule_loader.execute_custom_rules(
+        cfn_model,
+        @config.custom_rule_loader.rule_definitions
+      )
 
       violations = filter_violations_by_blacklist_and_profile(violations)
       violations = mark_line_numbers(violations, cfn_model)
-    rescue Psych::SyntaxError, ParserError => parser_error
-      violations << fatal_violation(parser_error.to_s)
+    rescue RuleRepoException, Psych::SyntaxError, ParserError => fatal_error
+      violations << fatal_violation(fatal_error.to_s)
     rescue JSON::ParserError => json_parameters_error
       error = "JSON Parameter values parse error: #{json_parameters_error}"
       violations << fatal_violation(error)

data/lib/cfn-nag/cfn_nag_config.rb

@@ -8,17 +8,20 @@ class CfnNagConfig
                  allow_suppression: true,
                  print_suppression: false,
                  isolate_custom_rule_exceptions: false,
-                 fail_on_warnings: false)
+                 fail_on_warnings: false,
+                 rule_repository_definitions: [])
     @rule_directory = rule_directory
     @custom_rule_loader = CustomRuleLoader.new(
       rule_directory: rule_directory,
       allow_suppression: allow_suppression,
       print_suppression: print_suppression,
-      isolate_custom_rule_exceptions: isolate_custom_rule_exceptions
+      isolate_custom_rule_exceptions: isolate_custom_rule_exceptions,
+      rule_repository_definitions: rule_repository_definitions
     )
     @profile_definition = profile_definition
     @blacklist_definition = blacklist_definition
     @fail_on_warnings = fail_on_warnings
+    @rule_repositories = rule_repositories
   end
   # rubocop:enable Metrics/ParameterLists
 
@@ -27,4 +30,5 @@ class CfnNagConfig
   attr_reader :profile_definition
   attr_reader :blacklist_definition
   attr_reader :fail_on_warnings
+  attr_reader :rule_repositories
 end

data/lib/cfn-nag/cfn_nag_executor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'trollop'
+require 'optimist'
 require 'cfn-nag/cli_options'
 require 'cfn-nag/cfn_nag_config'
 
@@ -10,6 +10,7 @@ class CfnNagExecutor
     @blacklist_definition = nil
     @parameter_values_string = nil
     @condition_values_string = nil
+    @rule_repository_definitions = []
   end
 
   def scan(options_type:)
@@ -73,26 +74,28 @@ class CfnNagExecutor
 
   def validate_options(opts)
     unless opts[:output_format].nil? || %w[colortxt txt json].include?(opts[:output_format])
-      Trollop.die(:output_format,
-                  'Must be colortxt, txt, or json')
+      Optimist.die(:output_format,
+                   'Must be colortxt, txt, or json')
     end
   end
 
   def execute_io_options(opts)
-    unless opts[:profile_path].nil?
-      @profile_definition = IO.read(opts[:profile_path])
-    end
+    @profile_definition = read_conditionally(opts[:profile_path])
 
-    unless opts[:blacklist_path].nil?
-      @blacklist_definition = IO.read(opts[:blacklist_path])
-    end
+    @blacklist_definition = read_conditionally(opts[:blacklist_path])
+
+    @parameter_values_string = read_conditionally(opts[:parameter_values_path])
 
-    unless opts[:parameter_values_path].nil?
-      @parameter_values_string = IO.read(opts[:parameter_values_path])
+    @condition_values_string = read_conditionally(opts[:condition_values_path])
+
+    opts[:rule_repository]&.each do |rule_repository|
+      @rule_repository_definitions << IO.read(rule_repository)
     end
+  end
 
-    unless opts[:condition_values_path].nil?
-      @condition_values_string = IO.read(opts[:condition_values_path])
+  def read_conditionally(path)
+    unless path.nil?
+      IO.read(path)
     end
   end
 
@@ -104,7 +107,8 @@ class CfnNagExecutor
       allow_suppression: opts[:allow_suppression],
       print_suppression: opts[:print_suppression],
       isolate_custom_rule_exceptions: opts[:isolate_custom_rule_exceptions],
-      fail_on_warnings: opts[:fail_on_warnings]
+      fail_on_warnings: opts[:fail_on_warnings],
+      rule_repository_definitions: @rule_repository_definitions
     )
   end
 

data/lib/cfn-nag/cli_options.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'trollop'
+require 'optimist'
 
 # rubocop:disable Metrics/ClassLength
 class Options
@@ -29,7 +29,7 @@ class Options
     custom_rule_exceptions_message = @custom_rule_exceptions_message
     version = @version
 
-    Trollop.options do
+    Optimist.options do
       usage options_message
       version version
 
@@ -87,6 +87,10 @@ class Options
           'Format of results: [txt, json, colortxt]',
           type: :string,
           default: 'colortxt'
+      opt :rule_repository,
+          'Path(s) to a rule repository to include in rule discovery',
+          type: :strings,
+          required: false
     end
   end
 
@@ -102,7 +106,7 @@ class Options
     custom_rule_exceptions_message = @custom_rule_exceptions_message
     version = @version
 
-    Trollop.options do
+    Optimist.options do
       version version
       opt :input_path,
           input_path_message,
@@ -167,6 +171,10 @@ class Options
           type: :boolean,
           required: false,
           default: false
+      opt :rule_repository,
+          'Path(s)s to rule repository to include in rule discovery',
+          type: :strings,
+          required: false
     end
   end
   # rubocop:enable Metrics/BlockLength

data/lib/cfn-nag/custom_rule_loader.rb

@@ -3,47 +3,46 @@
 require 'cfn-model'
 require 'logging'
 require_relative 'rule_registry'
-require 'cfn-nag/jmes_path_evaluator'
-require 'cfn-nag/jmes_path_discovery'
+require_relative 'rule_repos/file_based_rule_repo'
+require_relative 'rule_repos/gem_based_rule_repo'
+require_relative 'rule_repos/s3_based_rule_repo'
+require_relative 'rule_repository_loader'
 
 ##
 # This object can discover the internal and custom user-provided rules and
 # apply these rules to a CfnModel object
 #
-# rubocop:disable Metrics/ClassLength
 class CustomRuleLoader
   def initialize(rule_directory: nil,
                  allow_suppression: true,
                  print_suppression: false,
-                 isolate_custom_rule_exceptions: false)
+                 isolate_custom_rule_exceptions: false,
+                 rule_repository_definitions: [])
     @rule_directory = rule_directory
     @allow_suppression = allow_suppression
     @print_suppression = print_suppression
     @isolate_custom_rule_exceptions = isolate_custom_rule_exceptions
-    validate_extra_rule_directory rule_directory
+    @rule_repository_definitions = rule_repository_definitions
+    @registry = nil
   end
 
-  # rubocop:disable Security/Eval
-  def rule_definitions
-    rule_registry = RuleRegistry.new
-
-    discover_rule_classes(@rule_directory).each do |rule_class|
-      rule_registry
-        .definition(**rule_registry_from_rule_class(rule_class))
-    end
-
-    discover_jmespath_filenames(@rule_directory).each do |jmespath_file|
-      evaluator = JmesPathDiscovery.new rule_registry
-      evaluator.instance_eval do
-        eval IO.read jmespath_file
-      end
+  ##
+  # the first time this runs, it's "expensive".  the core rules, the gem-based rules will load, and
+  # any other repos like "s3" will go the expensive route.  after that, it's cached so you can
+  # call it as many times as you like unless you force_refresh
+  #
+  def rule_definitions(force_refresh: false)
+    if @registry.nil? || force_refresh
+      @registry = FileBasedRuleRepo.new(@rule_directory).discover_rules
+      @registry.merge! GemBasedRuleRepo.new.discover_rules
+
+      @registry = RuleRepositoryLoader.new.merge(@registry, @rule_repository_definitions)
+      @registry
     end
-
-    rule_registry
+    @registry
   end
-  # rubocop:enable Security/Eval
 
-  def execute_custom_rules(cfn_model)
+  def execute_custom_rules(cfn_model, rules_registry)
     if Logging.logger['log'].debug?
       Logging.logger['log'].debug "cfn_model: #{cfn_model}"
     end
@@ -52,37 +51,16 @@ class CustomRuleLoader
 
     validate_cfn_nag_metadata(cfn_model)
 
-    filter_rule_classes cfn_model, violations
-
-    filter_jmespath_filenames cfn_model, violations
+    filter_rule_classes cfn_model, violations, rules_registry
 
     violations
   end
 
   private
 
-  def rule_registry_from_rule_class(rule_class)
-    rule = rule_class.new
-    { id: rule.rule_id,
-      type: rule.rule_type,
-      message: rule.rule_text }
-  end
-
-  # rubocop:disable Security/Eval
-  def filter_jmespath_filenames(cfn_model, violations)
-    discover_jmespath_filenames(@rule_directory).each do |jmespath_file|
-      evaluator = JmesPathEvaluator.new cfn_model
-      evaluator.instance_eval do
-        eval IO.read jmespath_file
-      end
-      violations += evaluator.violations
-    end
-  end
-  # rubocop:enable Security/Eval
-
   # rubocop:disable Style/RedundantBegin
-  def filter_rule_classes(cfn_model, violations)
-    discover_rule_classes(@rule_directory).each do |rule_class|
+  def filter_rule_classes(cfn_model, violations, rules_registry)
+    rules_registry.rule_classes.each do |rule_class|
       begin
         filtered_cfn_model = cfn_model_with_suppressed_resources_removed(
           cfn_model: cfn_model,
@@ -169,51 +147,4 @@ class CustomRuleLoader
     end
     cfn_model
   end
-
-  def validate_extra_rule_directory(rule_directory)
-    return true if rule_directory.nil? || File.directory?(rule_directory)
-
-    raise "Not a real directory #{rule_directory}"
-  end
-
-  def discover_rule_filenames(rule_directory)
-    rule_filenames = []
-    unless rule_directory.nil?
-      rule_filenames += Dir[File.join(rule_directory, '*Rule.rb')].sort
-    end
-    rule_filenames += Dir[File.join(__dir__, 'custom_rules', '*Rule.rb')].sort
-    # Windows fix when running ruby from Command Prompt and not bash
-    rule_filenames.reject! { |filename| filename =~ /_rule.rb$/ }
-    Logging.logger['log'].debug "rule_filenames: #{rule_filenames}"
-    rule_filenames
-  end
-
-  def discover_rule_classes(rule_directory)
-    rule_classes = []
-
-    rule_filenames = discover_rule_filenames(rule_directory)
-
-    rule_filenames.each do |rule_filename|
-      require(rule_filename)
-      rule_classname = File.basename(rule_filename, '.rb')
-
-      rule_classes << Object.const_get(rule_classname)
-    end
-    Logging.logger['log'].debug "rule_classes: #{rule_classes}"
-
-    rule_classes
-  end
-
-  def discover_jmespath_filenames(rule_directory)
-    rule_filenames = []
-    unless rule_directory.nil?
-      rule_filenames += Dir[File.join(rule_directory, '*jmespath.rb')].sort
-    end
-    rule_filenames += Dir[File.join(__dir__,
-                                    'custom_rules',
-                                    '*jmespath.rb')].sort
-    Logging.logger['log'].debug "jmespath_filenames: #{rule_filenames}"
-    rule_filenames
-  end
 end
-# rubocop:enable Metrics/ClassLength

data/lib/cfn-nag/rule_dumper.rb

@@ -3,19 +3,23 @@
 require_relative 'custom_rule_loader'
 require_relative 'profile_loader'
 require_relative 'result_view/rules_view'
+require_relative 'rule_repository_loader'
 
 class CfnNagRuleDumper
   def initialize(profile_definition: nil,
                  rule_directory: nil,
-                 output_format: nil)
+                 output_format: nil,
+                 rule_repository_definitions: [])
     @rule_directory = rule_directory
     @profile_definition = profile_definition
     @output_format = output_format
+    @rule_repository_definitions = rule_repository_definitions
   end
 
   def dump_rules
-    custom_rule_loader = CustomRuleLoader.new(rule_directory: @rule_directory)
-    rule_registry = custom_rule_loader.rule_definitions
+    rule_registry = FileBasedRuleRepo.new(@rule_directory).discover_rules
+    rule_registry.merge! GemBasedRuleRepo.new.discover_rules
+    rule_registry = RuleRepositoryLoader.new.merge(rule_registry, @rule_repository_definitions)
 
     profile = nil
     unless @profile_definition.nil?

data/lib/cfn-nag/rule_registry.rb

@@ -1,33 +1,56 @@
 # frozen_string_literal: true
 
+require 'set'
 require_relative 'rule_definition'
 
+##
+# This registry contains all the discovered rule classes that are to be used
+# for inspection.
+#
+# Historically this just kept the metadata around the rules that percolated into
+# the violation reports.  This is no longer true as the rule discovery logic is being
+# split out to support multiple discover algorithms (i.e. rule repositories).
+# The CustomRuleLoader asks the discovery delegate to the rule classes, and that
+# discovery returns this object with the rule definitions, and the class itself
+# that can be invoked to do an inspection
+#
 class RuleRegistry
-  attr_reader :rules, :duplicate_ids
+  attr_reader :rules, :duplicate_ids, :rule_classes
 
   def initialize
     @rules = []
     @duplicate_ids = []
+    @rule_classes = Set.new
   end
 
   def duplicate_ids?
     @duplicate_ids.count.positive?
   end
 
-  def definition(id:,
-                 type:,
-                 message:)
-    rule_definition = RuleDefinition.new(id: id,
-                                         type: type,
-                                         message: message)
-    existing_def = by_id id
+  def merge!(other_rule_registry)
+    @rules += other_rule_registry.rules
+    @duplicate_ids += other_rule_registry.duplicate_ids
+    @rule_classes += other_rule_registry.rule_classes
+  end
+
+  def definition(rule_class)
+    @rule_classes.add rule_class
+
+    rule = rule_class.new
+
+    existing_def = by_id rule.rule_id
 
     if existing_def.nil?
+      rule_definition = RuleDefinition.new(
+        id: rule.rule_id,
+        type: rule.rule_type,
+        message: rule.rule_text
+      )
       add_rule rule_definition
     else
       @duplicate_ids << {
-        id: id,
-        new_message: message,
+        id: rule.rule_id,
+        new_message: rule.rule_text,
         registered_message: existing_def.message
       }
     end
@@ -37,6 +60,10 @@ class RuleRegistry
     @rules.find { |rule| rule.id == id }
   end
 
+  def ids
+    @rules.map(&:id)
+  end
+
   def warnings
     @rules.select { |rule| rule.type == RuleDefinition::WARNING }
   end
@@ -47,8 +74,8 @@ class RuleRegistry
 
   private
 
-  def add_rule(violation_def)
-    @rules << violation_def
-    violation_def
+  def add_rule(rule_definition)
+    @rules << rule_definition
+    rule_definition
   end
 end

data/lib/cfn-nag/rule_repo_exception.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class RuleRepoException < StandardError
+  def initialize(msg: 'Trouble loading a rule-repository')
+    super
+  end
+end

data/lib/cfn-nag/rule_repos/file_based_rule_repo.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'logging'
+
+##
+# This is really the traditional implementation for CustomRuleLoader
+# that looks in cfn-nag/custom_rules and an optional directory of a
+# client's choosing
+#
+class FileBasedRuleRepo
+  def initialize(rule_directory)
+    @rule_directory = rule_directory
+    validate_extra_rule_directory rule_directory
+  end
+
+  def discover_rules
+    rule_registry = RuleRegistry.new
+
+    # we look on the file system, and we load from the file system into a Class
+    # that the runtime can refer back to later from the registry which is effectively
+    # just a set of rule definitons
+    discover_rule_classes(@rule_directory).each do |rule_class|
+      rule_registry.definition(rule_class)
+    end
+
+    rule_registry
+  end
+
+  private
+
+  def validate_extra_rule_directory(rule_directory)
+    return true if rule_directory.nil? || File.directory?(rule_directory)
+
+    raise "Not a real directory #{rule_directory}"
+  end
+
+  def discover_rule_filenames(rule_directory)
+    rule_filenames = []
+    unless rule_directory.nil?
+      rule_filenames += Dir[File.join(rule_directory, '*Rule.rb')].sort
+    end
+    rule_filenames += Dir[File.join(__dir__, '..', 'custom_rules', '*Rule.rb')].sort
+
+    # Windows fix when running ruby from Command Prompt and not bash
+    rule_filenames.reject! { |filename| filename =~ /_rule.rb$/ }
+    Logging.logger['log'].debug "rule_filenames: #{rule_filenames}"
+    rule_filenames
+  end
+
+  def discover_rule_classes(rule_directory)
+    rule_classes = []
+
+    rule_filenames = discover_rule_filenames(rule_directory)
+    rule_filenames.each do |rule_filename|
+      require(rule_filename)
+      rule_classname = File.basename(rule_filename, '.rb')
+
+      rule_classes << Object.const_get(rule_classname)
+    end
+    Logging.logger['log'].debug "rule_classes: #{rule_classes}"
+
+    rule_classes
+  end
+end

data/lib/cfn-nag/rule_repos/gem_based_rule_repo.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'cfn-nag/base_rule'
+require 'rubygems'
+
+class GemBasedRuleRepo
+  def discover_rules
+    rule_registry = RuleRegistry.new
+
+    rule_gem_names.each do |rule_gem_name|
+      next unless load_gem_entrypoint(rule_gem_name)
+
+      gem_path = Gem.loaded_specs[rule_gem_name].full_gem_path
+      require_all_rb_files_in_gem gem_path, rule_gem_name
+    end
+
+    unless rule_gem_names.empty?
+      ObjectSpace.each_object do |object|
+        if derives_from_base_rule?(object) || (object.respond_to?(:cfn_nag_rule?) && object.cfn_nag_rule?)
+          rule_registry.definition(object)
+        end
+      end
+    end
+
+    rule_registry
+  end
+
+  private
+
+  def require_all_rb_files_in_gem(gem_path, rule_gem_name)
+    Dir.glob("#{gem_path}/lib/#{rule_gem_name}/**/*.rb").sort.each do |rule_file|
+      require rule_file
+    end
+  end
+
+  def rule_gem_names
+    rule_gem_specs.map(&:name)
+  end
+
+  def rule_gem_specs
+    specs = []
+
+    # https://github.com/rvm/rubygems-bundler can interfere here
+    # if you happen to have a gemspec in the current working directory?
+    # Bundle.setup runs and replaces a SpecSet in here instead of collection of Specification?
+    Gem::Specification.each do |spec|
+      specs << spec if spec.metadata && spec.metadata['cfn_nag_rules'] == 'true'
+    end
+    specs
+  end
+
+  def load_gem_entrypoint(rule_gem_name)
+    require rule_gem_name
+    true
+  rescue LoadError
+    STDERR.puts "Could not require #{rule_gem_name} - does the rule gem have a top level entry point?"
+    false
+  end
+
+  def derives_from_base_rule?(object)
+    object.respond_to?(:superclass) && object.superclass == CfnNag::BaseRule
+  end
+end

data/lib/cfn-nag/rule_repos/s3_based_rule_repo.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-s3'
+require 'lightly'
+require 'json'
+require_relative '../rule_registry'
+require_relative '../rule_repo_exception'
+
+class Object
+  ##
+  # This is meta-magic evil.  eval apparently has lexical scope so... opening up Object to
+  # evaluate ruby code that contains top-level Class definitions
+  #
+  # Without this, the class ends up "under" the scope of the class which in this case would be
+  # S3BucketBasedRuleRepo
+  #
+  # rubocop:disable Security/Eval
+  def eval_code_in_object_scope(code)
+    eval code
+  end
+  # rubocop:enable Security/Eval
+end
+
+class S3BucketBasedRuleRepo
+  def initialize(s3_bucket_name:, prefix:, index_lifetime: '1h', aws_profile: nil)
+    @s3_bucket_name = s3_bucket_name
+    @prefix = prefix
+    @index_cache = Lightly.new(
+      dir: cache_path('cfn_nag_s3_index_cache', s3_bucket_name),
+      life: index_lifetime,
+      hash: true
+    )
+
+    # except in dev mode, rules are immutable so once we have it don't worry about it changing
+    @rule_cache = Lightly.new(
+      dir: cache_path('cfn_nag_s3_rule_cache', s3_bucket_name),
+      life: '1000d',
+      hash: true
+    )
+    @aws_profile = aws_profile
+    @s3_resource = nil
+  end
+
+  def discover_rules
+    rule_registry = RuleRegistry.new
+
+    index = index(@s3_bucket_name, @prefix)
+    index.each do |rule_object_key|
+      rule_code = @rule_cache.get(rule_object_key) do
+        cache_miss(rule_object_key)
+      end
+
+      rule_class_name = select_class_name_from_object_key(rule_object_key)
+
+      eval_code_in_object_scope rule_code
+
+      rule_registry.definition(Object.const_get(rule_class_name))
+    end
+
+    rule_registry
+  end
+
+  def nuke_cache
+    cached_dirs = [
+      cache_path('cfn_nag_s3_index_cache', @s3_bucket_name),
+      cache_path('cfn_nag_s3_rule_cache', @s3_bucket_name)
+    ]
+    FileUtils.rm_rf(cached_dirs)
+  end
+
+  private
+
+  def cache_miss(key)
+    rule_code_record = s3_object_content(@s3_bucket_name, key)
+    rule_code_record.body.read
+  end
+
+  def cache_path(cache_name, s3_bucket_name)
+    "/tmp/#{cache_name}/#{s3_bucket_name}"
+  end
+
+  def s3_resource
+    return @s3_resource unless @s3_resource.nil?
+
+    @s3_resource = @aws_profile ? Aws::S3::Resource.new(profile: @aws_profile) : Aws::S3::Resource.new
+  end
+
+  def select_class_name_from_object_key(key)
+    key.split('/')[-1].gsub('.rb', '')
+  end
+
+  def index(s3_bucket_name, prefix)
+    index_json_str = @index_cache.get('index_cfn_nag') do
+      discover_rule_s3_objects(s3_bucket_name, prefix).to_json
+    end
+    JSON.parse(index_json_str)
+  end
+
+  def s3_object(s3_bucket_name, key)
+    rule_bucket = s3_resource.bucket(s3_bucket_name)
+    rule_bucket.object key
+  end
+
+  def s3_object_content(s3_bucket_name, key)
+    s3_object(s3_bucket_name, key).get
+  end
+
+  def discover_rule_s3_objects(s3_bucket_name, prefix)
+    rule_bucket = s3_resource.bucket(s3_bucket_name)
+    objects = rule_bucket.objects(prefix: prefix)
+    rule_objects = objects.select do |object|
+      object.key.match(/.*Rule\.rb/)
+    end
+    rule_objects.map(&:key)
+  rescue Aws::S3::Errors::NoSuchBucket
+    raise RuleRepoException.new(msg: "Rule bucket not found: #{s3_bucket_name}")
+  end
+end

data/lib/cfn-nag/rule_repository_loader.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require_relative 'rule_repo_exception'
+
+##
+# This captures logic for instantiating the RuleRepo implementations
+# and merging their rule registries.  This was baked into CustomRuleLoader
+# but broken out for cfn_nag_rules to use as well
+#
+class RuleRepositoryLoader
+  def merge(rule_registry, rule_repository_definitions)
+    rule_repository_definitions.each do |rule_repository_definition|
+      rule_registry.merge! rule_repository(rule_repository_definition).discover_rules
+    end
+    rule_registry
+  end
+
+  private
+
+  def to_sym_keys(hash)
+    sym_hash = {}
+    hash.each do |k, v|
+      sym_hash[k.to_sym] = v
+    end
+    sym_hash
+  end
+
+  def rule_repository(rule_repository_definition_str)
+    rule_repository_definition = YAML.safe_load rule_repository_definition_str
+    unless rule_repository_definition['repo_class_name']
+      raise RuleRepoException.new(msg: 'Malformed repo definition: missing repo_class_name')
+    end
+
+    repo_class = class_from_name(rule_repository_definition['repo_class_name'])
+    if rule_repository_definition['repo_arguments']&.is_a?(Hash)
+      repo_class.new(**to_sym_keys(rule_repository_definition['repo_arguments']))
+    else
+      repo_class.new
+    end
+  end
+
+  def class_from_name(name)
+    Object.const_get name
+  rescue NameError
+    raise RuleRepoException.new(msg: "Malformed repo definition: repo_class_name: #{name} not loaded")
+  end
+end

data/lib/cfn-nag/violation.rb

@@ -9,7 +9,7 @@ class Violation < RuleDefinition
   def initialize(id:,
                  type:,
                  message:,
-                 logical_resource_ids: nil,
+                 logical_resource_ids: [],
                  line_numbers: [])
     super id: id,
           type: type,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cfn-nag
 version: !ruby/object:Gem::Version
-  version: 0.4.82
+  version: 0.5.0
 platform: ruby
 authors:
 - Eric Kascic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-04 00:00:00.000000000 Z
+date: 2020-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -81,61 +81,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.4.14
 - !ruby/object:Gem::Dependency
-  name: jmespath
+  name: logging
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: 2.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: 2.2.2
 - !ruby/object:Gem::Dependency
-  name: logging
+  name: netaddr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.0.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.0.4
 - !ruby/object:Gem::Dependency
-  name: netaddr
+  name: optimist
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.4
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.4
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.60.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.60.1
 - !ruby/object:Gem::Dependency
-  name: trollop
+  name: lightly
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.2
+        version: 0.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.2
+        version: 0.3.2
 description: Auditing tool for CloudFormation templates
 email: 
 executables:
@@ -287,8 +301,6 @@ files:
 - lib/cfn-nag/custom_rules/password_base_rule.rb
 - lib/cfn-nag/custom_rules/sub_property_with_list_password_base_rule.rb
 - lib/cfn-nag/ip_addr.rb
-- lib/cfn-nag/jmes_path_discovery.rb
-- lib/cfn-nag/jmes_path_evaluator.rb
 - lib/cfn-nag/profile_loader.rb
 - lib/cfn-nag/result_view/colored_stdout_results.rb
 - lib/cfn-nag/result_view/json_results.rb
@@ -299,6 +311,11 @@ files:
 - lib/cfn-nag/rule_dumper.rb
 - lib/cfn-nag/rule_id_set.rb
 - lib/cfn-nag/rule_registry.rb
+- lib/cfn-nag/rule_repo_exception.rb
+- lib/cfn-nag/rule_repos/file_based_rule_repo.rb
+- lib/cfn-nag/rule_repos/gem_based_rule_repo.rb
+- lib/cfn-nag/rule_repos/s3_based_rule_repo.rb
+- lib/cfn-nag/rule_repository_loader.rb
 - lib/cfn-nag/template_discovery.rb
 - lib/cfn-nag/util/blank.rb
 - lib/cfn-nag/util/enforce_reference_parameter.rb

data/lib/cfn-nag/jmes_path_discovery.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-class JmesPathDiscovery
-  def initialize(rule_registry)
-    @rule_registry = rule_registry
-  end
-
-  def warning(id:, message:)
-    @rule_registry.definition(id: id,
-                              type: Violation::WARNING,
-                              message: message)
-  end
-
-  def failure(id:, message:)
-    @rule_registry.definition(id: id,
-                              type: Violation::FAILING_VIOLATION,
-                              message: message)
-  end
-end

data/lib/cfn-nag/jmes_path_evaluator.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require 'jmespath'
-require 'logging'
-
-##
-# THIS DOES NOT RESPECT SUPPRESSIONS!!!!!!
-##
-class JmesPathEvaluator
-  def initialize(cfn_model)
-    @cfn_model = cfn_model
-    @warnings = []
-    @failures = []
-  end
-
-  def warning(id:, jmespath:, message:)
-    violation id: id,
-              jmespath: jmespath,
-              message: message,
-              violation_type: Violation::WARNING
-  end
-
-  def failure(id:, jmespath:, message:)
-    violation id: id,
-              jmespath: jmespath,
-              message: message,
-              violation_type: Violation::FAILING_VIOLATION
-  end
-
-  def violations
-    @warnings + @failures
-  end
-
-  private
-
-  def violation(id:, jmespath:, message:, violation_type:)
-    Logging.logger['log'].debug jmespath
-
-    logical_resource_ids = JMESPath.search(jmespath,
-                                           flatten(@cfn_model.raw_model))
-
-    unless logical_resource_ids.empty?
-      @warnings << Violation.new(id: id,
-                                 type: violation_type,
-                                 message: message,
-                                 logical_resource_ids: logical_resource_ids)
-    end
-  end
-
-  def flatten(hash)
-    hash['Resources'].each do |logical_resource_id, resource|
-      resource['id'] = logical_resource_id
-    end
-    hash
-  end
-end