abide_dev_utils 0.10.1 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c47baf812f2bb3b64951dc87b3aa1886eca947f0bf3a57bbed12b7ed7fda97ab
-  data.tar.gz: 4488f609550b251474337f1574c5c216601035fc5bba1fc9acb76e278252b5b5
+  metadata.gz: 7cf13209260261bcf509e7953d4af4a47aa7301cc7c2e8227c3d736d163cd0b1
+  data.tar.gz: a6d6cb52aadf58e33ce6a57d77c7921226ebb130aa2d6e4350c42c197df0f06c
 SHA512:
-  metadata.gz: f4e517c44f1c728689d1de11d0e306f65352ecf1b0507671d588ac5eb21dc294ac7bca68e7c790f21b9a47fa565ed775c2a28ce45257395203a0fa296c6f4389
-  data.tar.gz: 2b5ff4f1ad40d4000bb93862902c1a89e4620632825d19a34cd118fa43cc15823132cb4e9d246a1655ad1f6bd9636581398651e9082e8b16129f0c79eba9a5df
+  metadata.gz: be0ac34b2ee3ea116793580ef1ffff41ca4f11b41ceb6587866183734f1aee15a566b5a0063b106a18962739a1d4d7091d6e8d276dffdf6101cfcfb48afb3832
+  data.tar.gz: 97f9b753cb94f1a1d326eb370a59ae35d00a3c2797b6e030941e51f51e9f7713eca0af9148f9858a0f4c375273bb468ab1df928cd589903d48b9bbe7c86a92fd

data/.gitignore

@@ -4,6 +4,7 @@
 /coverage/
 /doc/
 /pkg/
+/spec/fixtures/
 /spec/reports/
 /tmp/
 w10_20h2.xml

data/.rubocop.yml

@@ -39,6 +39,9 @@ Style/IpAddresses:
   Exclude:
   - spec/rubocop/cop/style/ip_addresses_spec.rb
 
+Style/RegexpLiteral:
+  Enabled: false
+
 Layout/EndOfLine:
   EnforcedStyle: lf
 
@@ -133,4 +136,7 @@ Performance/CollectionLiteralInLoop:
     - 'spec/**/*.rb'
 
 RSpec/StubbedMock:
-  Enabled: false
+  Enabled: false
+
+Convention/ExplicitBlockArgument:
+  Enabled: false

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    abide_dev_utils (0.10.1)
+    abide_dev_utils (0.11.0)
       amatch (~> 0.4)
       cmdparse (~> 3.0)
+      facterdb (>= 1.18)
       google-cloud-storage (~> 1.34)
       hashdiff (~> 1.0)
       jira-ruby (~> 2.2)
@@ -62,6 +63,9 @@ GEM
     facter (4.2.10)
       hocon (~> 1.3)
       thor (>= 1.0.1, < 2.0)
+    facterdb (1.18.0)
+      facter (< 5.0.0)
+      jgrep
     faraday (2.3.0)
       faraday-net_http (~> 2.0)
       ruby2_keywords (>= 0.0.4)
@@ -120,6 +124,7 @@ GEM
     httpclient (2.8.3)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
+    jgrep (1.5.4)
     jira-ruby (2.2.0)
       activesupport
       atlassian-jwt
@@ -138,8 +143,10 @@ GEM
     nio4r (2.5.8)
     nokogiri (1.13.6-x86_64-darwin)
       racc (~> 1.4)
+    nokogiri (1.13.6-x86_64-linux)
+      racc (~> 1.4)
     oauth (0.5.10)
-    octokit (4.23.0)
+    octokit (4.25.0)
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
     os (1.1.4)
@@ -159,6 +166,17 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (4.0.7)
+    puppet (7.17.0)
+      concurrent-ruby (~> 1.0)
+      deep_merge (~> 1.0)
+      facter (> 2.0.1, < 5)
+      fast_gettext (>= 1.1, < 3)
+      hiera (>= 3.2.1, < 4)
+      locale (~> 2.1)
+      multi_json (~> 1.10)
+      puppet-resource_api (~> 1.5)
+      scanf (~> 1.0)
+      semantic_puppet (~> 1.0)
     puppet (7.17.0-universal-darwin)
       CFPropertyList (~> 2.2)
       concurrent-ruby (~> 1.0)
@@ -219,7 +237,7 @@ GEM
     ruby_parser (3.19.1)
       sexp_processor (~> 4.16)
     rubyzip (2.3.2)
-    sawyer (0.9.1)
+    sawyer (0.9.2)
       addressable (>= 2.3.5)
       faraday (>= 0.17.3, < 3)
     scanf (1.0.0)
@@ -249,6 +267,7 @@ GEM
 
 PLATFORMS
   x86_64-darwin-19
+  x86_64-linux
 
 DEPENDENCIES
   abide_dev_utils!

data/Rakefile

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'rake'
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 
@@ -10,3 +11,30 @@ require "rubocop/rake_task"
 RuboCop::RakeTask.new
 
 task default: %i[spec rubocop]
+
+namespace 'cem' do
+  directory 'spec/fixtures'
+
+  directory 'spec/fixtures/puppetlabs-cem_linux' do
+    sh 'git clone git@github.com:puppetlabs/puppetlabs-cem_linux.git spec/fixtures/puppetlabs-cem_linux'
+  end
+  file 'spec/fixtures/puppetlabs-cem_linux' => ['spec/fixtures']
+
+  directory 'spec/fixtures/puppetlabs-cem_windows' do
+    sh 'git clone git@github.com:puppetlabs/puppetlabs-cem_windows.git spec/fixtures/puppetlabs-cem_windows'
+  end
+  file 'spec/fixtures/puppetlabs-cem_windows' => ['spec/fixtures']
+
+  task :fixture, [:cem_mod] do |_, args|
+    case args.cem_mod
+    when /linux/
+      Rake::Task['spec/fixtures/puppetlabs-cem_linux'].invoke
+    when /windows/
+      Rake::Task['spec/fixtures/puppetlabs-cem_windows'].invoke
+    else
+      raise "Unknown CEM module #{args.cem_mod}"
+    end
+  end
+
+  multitask fixtures: %w[spec/fixtures/puppetlabs-cem_linux spec/fixtures/puppetlabs-cem_windows]
+end

data/abide_dev_utils.gemspec

@@ -41,6 +41,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'google-cloud-storage', '~> 1.34'
   spec.add_dependency 'hashdiff', '~> 1.0'
   spec.add_dependency 'amatch', '~> 0.4'
+  spec.add_dependency 'facterdb', '>= 1.18'
 
   # Dev dependencies
   spec.add_development_dependency 'bundler'

data/lib/abide_dev_utils/cem/benchmark.rb

@@ -0,0 +1,291 @@
+# frozen_string_literal: true
+
+require 'abide_dev_utils/errors'
+require 'abide_dev_utils/ppt/facter_utils'
+require 'abide_dev_utils/cem/mapping/mapper'
+
+module AbideDevUtils
+  module CEM
+    # Repesents a benchmark for purposes of organizing data for markdown representation
+    class Benchmark
+      attr_reader :osname, :major_version, :os_facts, :osfamily, :hiera_conf, :module_name, :framework, :rules
+
+      def initialize(osname, major_version, hiera_conf, module_name, framework: 'cis')
+        @osname = osname
+        @major_version = major_version
+        @os_facts = AbideDevUtils::Ppt::FacterUtils.recursive_facts_for_os(@osname, @major_version)
+        @osfamily = @os_facts['os']['family']
+        @hiera_conf = hiera_conf
+        @module_name = module_name
+        @framework = framework
+        @rules = {}
+        @map_cache = {}
+        @rules_in_map = {}
+        load_rules
+      end
+
+      # Creates Benchmark objects from a Puppet module
+      # @param pupmod [AbideDevUtils::Ppt::PuppetModule] A PuppetModule instance
+      # @param skip_errors [Boolean] True skips errors and loads non-erroring benchmarks, false raises the error.
+      # @return [Array<AbideDevUtils::CEM::Benchmark>] Array of Benchmark instances
+      def self.benchmarks_from_puppet_module(pupmod, ignore_all_errors: false, ignore_framework_mismatch: true)
+        frameworks = pupmod.hiera_conf.local_hiera_files(hierarchy_name: 'Mapping Data').each_with_object([]) do |hf, ary|
+          parts = hf.path.split(pupmod.hiera_conf.default_datadir)[-1].split('/')
+          ary << parts[2] unless ary.include?(parts[2])
+        end
+        pupmod.supported_os.each_with_object([]) do |supp_os, ary|
+          osname, majver = supp_os.split('::')
+          if majver.is_a?(Array)
+            majver.sort.each do |v|
+              frameworks.each do |fw|
+                benchmark = Benchmark.new(osname, v, pupmod.hiera_conf, pupmod.name(strip_namespace: true), framework: fw)
+                ary << benchmark
+              rescue StandardError => e
+                raise e unless ignore_all_errors || (e.instance_of?(AbideDevUtils::Errors::MappingDataFrameworkMismatchError) && ignore_framework_mismatch)
+              end
+            end
+          else
+            frameworks.each do |fw|
+              benchmark = Benchmark.new(osname, majver, pupmod.hiera_conf, pupmod.name(strip_namespace: true), framework: fw)
+              ary << benchmark
+            rescue StandardError => e
+              raise e unless ignore_all_errors || (e.instance_of?(AbideDevUtils::Errors::MappingDataFrameworkMismatchError) && ignore_framework_mismatch)
+            end
+          end
+        end
+      end
+
+      def mapper
+        @mapper ||= AbideDevUtils::CEM::Mapping::Mapper.new(module_name, framework, load_mapping_data)
+      end
+
+      def map_data
+        mapper.map_data
+      end
+
+      def resource_data
+        @resource_data ||= load_resource_data
+      end
+
+      def title
+        mapper.title
+      end
+
+      def version
+        mapper.version
+      end
+
+      def title_key
+        @title_key ||= "#{title} #{version}"
+      end
+
+      def add_rule(rule_hash)
+        @rules << rule_hash
+      end
+
+      def rules_in_map(mtype, level: nil, profile: nil)
+        real_mtype = map_type(mtype)
+        cache_key = [real_mtype, level, profile].compact.join('-')
+        return @rules_in_map[cache_key] if @rules_in_map.key?(cache_key)
+
+        all_rim = mapper.each_with_array_like(real_mtype) do |(lvl, profs), arr|
+          next if lvl == 'benchmark' || (!level.nil? && lvl != level)
+
+          profs.each do |prof, maps|
+            next if !profile.nil? && prof != profile
+
+            # CIS and STIG differ in that STIG does not have profiles
+            control_ids = maps.respond_to?(:keys) ? maps.keys : prof
+            arr << control_ids
+          end
+        end
+        @rules_in_map[cache_key] = all_rim.flatten.uniq
+        @rules_in_map[cache_key]
+      end
+
+      def map(control_id, level: nil, profile: nil)
+        mapper.get(control_id, level: level, profile: profile)
+      end
+
+      def map_type(control_id)
+        mapper.map_type(control_id)
+      end
+
+      private
+
+      def load_rules
+        resource_data["#{module_name}::resources"].each do |_, rdata|
+          unless rdata.key?('controls')
+            puts "Controls key not found in #{rdata}"
+            next
+          end
+          rdata['controls'].each do |control, control_data|
+            rule_title = map(control)
+            next if rule_title.nil? || rule_title.empty?
+
+            rule_title.find { |id| map_type(id) == 'title' }
+            alternate_ids = map(rule_title)
+
+            next unless rule_title.is_a?(String)
+
+            @rules[rule_title] = {} unless @rules&.key?(rule_title)
+            @rules[rule_title]['number'] = alternate_ids.find { |id| map_type(id) == 'number' }
+            @rules[rule_title]['alternate_ids'] = alternate_ids
+            @rules[rule_title]['params'] = [] unless @rules[rule_title].key?('params')
+            @rules[rule_title]['level'] = [] unless @rules[rule_title].key?('level')
+            @rules[rule_title]['profile'] = [] unless @rules[rule_title].key?('profile')
+            param_hashes(control_data).each do |param_hash|
+              next if @rules[rule_title]['params'].include?(param_hash[:name])
+
+              @rules[rule_title]['params'] << param_hash
+            end
+            levels, profiles = find_levels_and_profiles(control)
+            unless @rules[rule_title]['level'] == levels
+              @rules[rule_title]['level'] = @rules[rule_title]['level'] | levels
+            end
+            unless @rules[rule_title]['profile'] == profiles
+              @rules[rule_title]['profile'] = @rules[rule_title]['profile'] | profiles
+            end
+            @rules[rule_title]['resource'] = rdata['type']
+          end
+        end
+        @rules = sort_rules
+      end
+
+      def param_hashes(control_data)
+        return [] if control_data.nil? || control_data.empty?
+
+        p_hashes = []
+        if !control_data.respond_to?(:each) && control_data == 'no_params'
+          p_hashes << no_params
+        else
+          control_data.each do |param, param_val|
+            p_hashes << {
+              name: param,
+              type: ruby_class_to_puppet_type(param_val.class.to_s),
+              default: param_val,
+            }
+          end
+        end
+        p_hashes
+      end
+
+      def no_params
+        { name: 'No parameters', type: nil, default: nil }
+      end
+
+      # We sort the rules by their control number so they
+      # appear in the REFERENCE in benchmark order
+      def sort_rules
+        sorted = @rules.dup.sort_by do |_, v|
+          control_num_to_int(v['number'])
+        end
+        sorted.to_h
+      end
+
+      # In order to sort the rules by their control number,
+      # we need to convert the control number to an integer.
+      # This is a rough conversion, but should be sufficient
+      # for the purposes of sorting. The multipliers are
+      # the 20th, 15th, 10th, and 5th numbers in the Fibonacci
+      # sequence, then 1 after that. The reason for this is to
+      # ensure a "spiraled" wieghting of the sections in the control
+      # number, with 1st section having the most sorting weight, 2nd
+      # having second most, etc. However, the differences in the multipliers
+      # are such that it would be difficult for the product of a lesser-weighted
+      # section to be greater than a greater-weighted section.
+      def control_num_to_int(control_num)
+        multipliers = [6765, 610, 55, 5, 1]
+        nsum = 0
+        midx = 0
+        control_num.split('.').each do |num|
+          multiplier = midx >= multipliers.length ? 1 : multipliers[midx]
+          nsum += num.to_i * multiplier
+          midx += 1
+        end
+        nsum
+      end
+
+      def find_levels_and_profiles(control_id)
+        levels = []
+        profiles = []
+        mapper.each_like(control_id) do |lvl, profile_hash|
+          next if lvl == 'benchmark'
+
+          profile_hash.each do |prof, _|
+            unless map(control_id, level: lvl, profile: prof).nil?
+              levels << lvl
+              profiles << prof
+            end
+          end
+        end
+        [levels, profiles]
+      end
+
+      def ruby_class_to_puppet_type(class_name)
+        pup_type = class_name.split('::').last.capitalize
+        case pup_type
+        when %r{(Trueclass|Falseclass)}
+          'Boolean'
+        when %r{(String|Pathname)}
+          'String'
+        when %r{(Integer|Fixnum)}
+          'Integer'
+        when %r{(Float|Double)}
+          'Float'
+        else
+          pup_type
+        end
+      end
+
+      def load_mapping_data
+        files = case module_name
+                when /_windows$/
+                  cem_windows_mapping_files
+                when /_linux$/
+                  cem_linux_mapping_files
+                else
+                  raise "Module name '#{module_name}' is not a CEM module"
+                end
+        validate_mapping_files_framework(files).each_with_object({}) do |f, h|
+          next unless f.path.include?(framework)
+
+          h[File.basename(f.path, '.yaml')] = YAML.load_file(f.path)
+        end
+      end
+
+      def cem_linux_mapping_files
+        facts = [['os.name', osname], ['os.release.major', major_version]]
+        mapping_files = hiera_conf.local_hiera_files_with_facts(*facts, hierarchy_name: 'Mapping Data')
+        raise AbideDevUtils::Errors::MappingFilesNotFoundError, facts if mapping_files.nil? || mapping_files.empty?
+
+        mapping_files
+      end
+
+      def cem_windows_mapping_files
+        facts = ['os.release.major', major_version]
+        mapping_files = hiera_conf.local_hiera_files_with_fact(facts[0], facts[1], hierarchy_name: 'Mapping Data')
+        raise AbideDevUtils::Errors::MappingFilesNotFoundError, facts if mapping_files.nil? || mapping_files.empty?
+
+        mapping_files
+      end
+
+      def validate_mapping_files_framework(files)
+        validated_files = files.select { |f| f.path_parts.include?(framework) }
+        if validated_files.nil? || validated_files.empty?
+          raise AbideDevUtils::Errors::MappingDataFrameworkMismatchError, framework
+        end
+
+        validated_files
+      end
+
+      def load_resource_data
+        facts = [['os.family', osfamily], ['os.name', osname], ['os.release.major', major_version]]
+        rdata_files = hiera_conf.local_hiera_files_with_facts(*facts, hierarchy_name: 'Resource Data')
+        raise AbideDevUtils::Errors::ResourceDataNotFoundError, facts if rdata_files.nil? || rdata_files.empty?
+
+        YAML.load_file(rdata_files[0].path)
+      end
+    end
+  end
+end