inspec_tools 2.0.2.pre13 → 2.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4dc5b1e3bca7c7388c9fd078810f362b74f79f5713fa5bd84b35837565ba0818
-  data.tar.gz: ee8cd675fdd5ed0d302f88f9ebb19e83f98e5d346bcce10c435655f297075b98
+  metadata.gz: c551987d4bfca9ae4d14630f4beb3f4dd6cbe78314a2f229d5df49b4c72cd0e2
+  data.tar.gz: d0553cb380d6103f21b879fdb43d39c502ba94d80cc6b30ae8c69cf246464f4d
 SHA512:
-  metadata.gz: ad7185fecbd5c6f122738af832e73f3c2571ba9b871901d17cd5c9fda124baa433bdd1747f77340c65b1da69b83a521d390dff0bd17c20b1ce4621e4cdc95088
-  data.tar.gz: 1034252e96e4212d31fd8ff28a18189e3707c6cb65e88343573cc24f18da8a33bbbaf90ab812e85034449b655bf1a9736458a31a72d2f92f400578827697d04c
+  metadata.gz: ffb7fa2a36d380a08c193fc0b1295c74a9479ef54ac5085cf3f36072509d67d235603c6fb589e81ce6c1d52544793fab3b91d37153e303923e7c99bb843f4510
+  data.tar.gz: 8e10eacec2b81f6a7c57198cc817050ee501ede1a390e347be50c80429d3834d506eb1f417b9f659ef0c620240d36e15cd65e118ea59fbbaa5481131cba4eb38

data/README.md

@@ -65,7 +65,7 @@ For Docker usage, replace the `inspec_tools` command with the correct Docker com
 
  - **On Linux and Mac**: `docker run -it -v$(pwd):/share mitre/inspec_tools`
  - **On Windows CMD**: `docker run -it -v%cd%:/share mitre/inspec_tools`
- 
+
 Note that all of the above Docker commands will mount your current directory on the Docker container. Ensure that you have navigated to the directory you intend to convert files in before executing the command.
 
 ### generate_map
@@ -98,6 +98,8 @@ USAGE: inspec_tools generate_inspec_metadata
 
 If the specified threshold is not met, an error code (1) is returned along with non-compliant elements.
 
+The compliance score are rounded down to the nearest whole number. For example a score of 77.3 would be displayed as 77.
+
 ```
 USAGE:  inspec_tools compliance [OPTIONS] -j <inspec-json> -i <threshold-inline>
 	inspec_tools compliance [OPTIONS] -j <inspec-json> -f <threshold-file>
@@ -142,22 +144,57 @@ failed.high.max: 1
 
 ## summary
 
-`summary` parses an inspec results json to create a summary json
+`summary` parses an inspec results json and displays the information from all of the tests that were run. Running the command with flags but `-j` it will display information like:
+
+```
+Overall compliance: 77%
+
+failed
+	total : 41
+	critical : 0
+	high : 3
+	medium : 33
+	low : 5
+passed
+	total : 174
+	critical : 0
+	high : 21
+	medium : 147
+	low : 6
+no_impact
+	total : 21
+	critical : 0
+	high : 0
+	medium : 0
+	low : 0
+skipped
+	total : 10
+	critical : 0
+	high : 2
+	medium : 5
+	low : 3
+error
+	total : 0
+	critical : 0
+	high : 0
+	medium : 0
+	low : 0
+```
+
+Using additional flags will override the normal output and only display the output that flag specifies.
+
+USAGE: inspec_tools summary [OPTIONS] -j <inspec-json>
 
 ```
-USAGE: inspec_tools summary [OPTIONS] -j <inspec-json> -o <summary-csv>
-
 FLAGS:
-	-j --inspec-json <inspec-json>   : path to InSpec results JSON
-	-o --output <output-json> 		   : path to summary JSON
-  -c --cli, --no-cli               : print formatted summary to STDOUT
+  -j --inspec-json <inspec-json>   : path to InSpec results JSON
   -V --verbose, --no-verbose       : print verbose an debug output
   -f --json-full, --no-json-full   : print the summary STDOUT as JSON
   -k --json-counts, --no-json_cou  : print the reslut status to STDOUT as JSON
 
 Examples:
 
-  inspec_tools summary -j examples/sample_json/rhel-simp.json -f -o summary.json -c
+  inspec_tools summary -j examples/sample_json/rhel-simp.json -f
 ```
 
 ## xccdf2inspec
@@ -174,7 +211,7 @@ FLAGS:
 	-f --format [ruby | hash]          : the format you would like (default: ruby) [optional]
 	-s --separate-files [true | false] : output the resulting controls as one or mutiple files (default: true) [optional]
 	-m --metadata <metadata-json>      : path to json file with additional metadata for the inspec.yml file [optional]
-	-r --replace-tags <array>          : A case-sensitive, comma separated list to replace tags with a $ if found in a group rules description tag [optional]
+	-r --replace-tags <array>          : A case-sensitive, space separated list to replace tags with a $ if found in a group rules description tag [optional]
 
 example: inspec_tools xccdf2inspec -x xccdf_file.xml -a attributes.yml -o myprofile -f ruby -s false
 ```
@@ -258,7 +295,7 @@ FLAGS:
 	-s --separate-files [true | false] : output the resulting controls as multiple files (default: true) [optional]
 	-d --debug                         : debug run [optional]
 
-example: inspec_tools pdf2inspec -p benchmark.pdf -o /path/to/myprofile -f ruby -s true
+example: inspec_tools pdf2inspec -p examples/CIS_Ubuntu_Linux_16.04_LTS_Benchmark_v1.0.0.pdf -o /path/to/myprofile -f ruby -s true
 ```
 
 ## xlsx2inspec

data/Rakefile

@@ -1,10 +1,9 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
+require 'rake/testtask'
 require File.expand_path('../lib/inspec_tools/version', __FILE__)
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
+  t.libs << 'test'
+  t.libs << 'lib'
   t.test_files = FileList['test/**/*_test.rb']
 end
 
@@ -22,9 +21,76 @@ namespace :test do
   end
 end
 
-desc 'Build and publish the gem'
-task publish: :build do
-  system("gem push pkg/inspec_tools-#{InspecTools::VERSION}.gem")
+desc 'Build for release'
+task :build_release do
+
+  Rake::Task["generate_mapping_objects"].reenable
+  Rake::Task["generate_mapping_objects"].invoke
+
+  system('gem build inspec_tools.gemspec')
 end
 
-task :default => :test
+desc 'Generate mapping objects'
+task :generate_mapping_objects do
+  require 'roo'
+
+  nist_mapping_cis_controls = ENV['NIST_MAPPING_CIS_CONTROLS'] || 'NIST_Map_02052020_CIS_Controls_Version_7.1_Implementation_Groups_1.2.xlsx'.freeze
+  nist_mapping_cis_critical_controls = ENV['NIST_MAPPING_CIS_CRITICAL_CONTROLS'] || 'NIST_Map_09212017B_CSC-CIS_Critical_Security_Controls_VER_6.1_Excel_9.1.2016.xlsx'.freeze
+
+  data_root_path = File.join(File.expand_path(__dir__), 'lib', 'data')
+  cis_controls_path = File.join(data_root_path, nist_mapping_cis_controls)
+  cis_critical_controls_path = File.join(data_root_path, nist_mapping_cis_critical_controls)
+
+  raise "#{cis_controls_path} does not exist" unless File.exist?(cis_controls_path)
+
+  raise "#{cis_critical_controls_path} does not exist" unless File.exist?(cis_critical_controls_path)
+
+  marshal_cis_controls(cis_controls_path, data_root_path)
+  marshal_cis_critical_controls(cis_critical_controls_path, data_root_path)
+end
+
+def marshal_cis_controls(cis_controls_path, data_root_path)
+  cis_to_nist = {}
+  Roo::Spreadsheet.open(cis_controls_path).sheet(3).each do |row|
+    if row[3].is_a?(Numeric)
+      cis_to_nist[row[3].to_s] = row[0]
+    else
+      cis_to_nist[row[2].to_s] = row[0] unless (row[2] == '') || row[2].to_i.nil?
+    end
+  end
+  output_file = File.new(File.join(data_root_path, 'cis_to_nist_mapping'), 'w')
+  Marshal.dump(cis_to_nist, output_file)
+  output_file.close
+end
+
+def marshal_cis_critical_controls(cis_critical_controls_path, data_root_path)
+  controls_spreadsheet = Roo::Spreadsheet.open(cis_critical_controls_path)
+  controls_spreadsheet.default_sheet = 'VER 6.1 Controls'
+  headings = {}
+  controls_spreadsheet.row(3).each_with_index { |header, idx| headings[header] = idx }
+
+  nist_ver = 4
+  cis_ver = controls_spreadsheet.row(2)[4].split(' ')[-1]
+  control_count = 1
+  mapping = []
+  ((controls_spreadsheet.first_row + 3)..controls_spreadsheet.last_row).each do |row_value|
+    current_row = {}
+    if controls_spreadsheet.row(row_value)[headings['NIST SP 800-53 Control #']].to_s != ''
+      current_row[:nist] = controls_spreadsheet.row(row_value)[headings['NIST SP 800-53 Control #']].to_s
+    else
+      current_row[:nist] = 'Not Mapped'
+    end
+    current_row[:nist_ver] = nist_ver
+    if controls_spreadsheet.row(row_value)[headings['Control']].to_s == ''
+      current_row[:cis] = control_count.to_s
+      control_count += 1
+    else
+      current_row[:cis] = controls_spreadsheet.row(row_value)[headings['Control']].to_s
+    end
+    current_row[:cis_ver] = cis_ver
+    mapping << current_row
+  end
+  output_file = File.new(File.join(data_root_path, 'cis_to_nist_critical_controls'), 'w')
+  Marshal.dump(mapping, output_file)
+  output_file.close
+end

data/lib/data/rubocop.yml

@@ -0,0 +1,4 @@
+AllCops:
+  DisabledByDefault: true
+Style/StringLiterals:
+  Enabled: true

data/lib/happy_mapper_tools/stig_attributes.rb

@@ -38,6 +38,7 @@ module HappyMapperTools
       element :documentable, Boolean, tag: 'Documentable'
       element :mitigations, String, tag: 'Mitigations'
       element :severity_override_guidance, String, tag: 'SeverityOverrideGuidance'
+      element :security_override_guidance, String, tag: 'SecurityOverrideGuidance'
       element :potential_impacts, String, tag: 'PotentialImpacts'
       element :third_party_tools, String, tag: 'ThirdPartyTools'
       element :mitigation_controls, String, tag: 'MitigationControl'
@@ -53,7 +54,8 @@ module HappyMapperTools
 
       detail_tags = %i(vuln_discussion false_positives false_negatives documentable
                        mitigations severity_override_guidance potential_impacts
-                       third_party_tools mitigation_controls responsibility ia_controls)
+                       third_party_tools mitigation_controls responsibility ia_controls
+                       security_override_guidance)
 
       detail_tags.each do |name|
         define_method name do
@@ -140,57 +142,75 @@ module HappyMapperTools
     end
 
     class DescriptionDetailsType
-      def self.type
-        DescriptionDetails
-      end
+      class << self
+        def type
+          DescriptionDetails
+        end
 
-      def self.apply(value) # rubocop:disable Metrics/AbcSize
-        value = value.gsub('&', 'and')
-        DescriptionDetails.parse "<Details>#{value}</Details>"
-      rescue Nokogiri::XML::SyntaxError
-        allowed_tags = %w{VulnDiscussion FalsePositives FalseNegatives Documentable
-                          Mitigations SeverityOverrideGuidance PotentialImpacts
-                          PotentialImpacts ThirdPartyTools MitigationControl
-                          Responsibility IAControls}
-
-        tags_found = value.scan(%r{(?<=<)([^\/]*?)((?= \/>)|(?=>))}).to_a
-
-        tags_found = tags_found.uniq.flatten.reject!(&:empty?)
-        offending_tags = tags_found - allowed_tags
-
-        if offending_tags.count > 1
-          puts "\n\nThe non-standard tags: #{offending_tags.to_s.colorize(:red)}" \
-               ' were found in: ' + "\n\n#{value}"
-        else
-          puts "\n\nThe non-standard tag: #{offending_tags.to_s.colorize(:red)}" \
-               ' was found in: ' + "\n\n#{value}"
+        def apply(value)
+          value = value.gsub('&', 'and')
+          DescriptionDetails.parse "<Details>#{value}</Details>"
+        rescue Nokogiri::XML::SyntaxError => e
+          if e.to_s.include?('StartTag')
+            report_invalid_start_tag(value, e)
+          else
+            report_disallowed_tags(value)
+          end
+        end
+
+        def apply?(value, _convert_to_type)
+          value.is_a?(String)
         end
-        puts "\n\nPlease:\n "
-        option_one = '(1) ' + '(best)'.colorize(:green) + ' Use the ' +
-                     '`-r --replace-tags array` '.colorize(:light_yellow) +
-                     '(case sensitive) option to replace the offending tags ' \
-                     'during processing of the XCCDF ' \
-                     'file to use the ' +
-                     "`$#{offending_tags[0]}` " .colorize(:light_green) +
-                     'syntax in your InSpec profile.'
-        option_two = '(2) Update your XCCDF file to *not use* non-standard XCCDF ' \
-                     'elements within ' +
-                     '`&lt;`,`&gt;`, `<` '.colorize(:red) +
-                     'or '.colorize(:default) +
-                     '`>` '.colorize(:red) +
-                     'as "placeholders", and use something that doesn\'t confuse ' \
-                     'the XML parser, such as : ' +
-                     "`$#{offending_tags[0]}`" .colorize(:light_green)
-        puts option_one
-        puts "\n"
-        puts option_two
-        # exit
-      end
 
-      def self.apply?(value, _convert_to_type)
-        value.is_a?(String)
+        private
+
+        def report_invalid_start_tag(value, error)
+          puts error.to_s.colorize(:red)
+          column = error.column - '<Details>'.length - 2
+          puts "Error around #{value[column-10..column+10].colorize(:light_yellow)}"
+          exit(1)
+        end
+
+        def report_disallowed_tags(value)
+          allowed_tags = %w{VulnDiscussion FalsePositives FalseNegatives Documentable
+                            Mitigations SeverityOverrideGuidance PotentialImpacts
+                            PotentialImpacts ThirdPartyTools MitigationControl
+                            Responsibility IAControl SecurityOverrideGuidance}
+
+          tags_found = value.scan(%r{(?<=<)([^\/]*?)((?= \/>)|(?=>))}).to_a
+
+          tags_found = tags_found.uniq.flatten.reject!(&:empty?)
+          offending_tags = tags_found - allowed_tags
+
+          if offending_tags.count > 1
+            puts "\n\nThe non-standard tags: #{offending_tags.to_s.colorize(:red)}" \
+                 ' were found in: ' + "\n\n#{value}"
+          else
+            puts "\n\nThe non-standard tag: #{offending_tags.to_s.colorize(:red)}" \
+                 ' was found in: ' + "\n\n#{value}"
+          end
+          puts "\n\nPlease:\n "
+          option_one = '(1) ' + '(best)'.colorize(:green) + ' Use the ' +
+                       '`-r --replace-tags array` '.colorize(:light_yellow) +
+                       '(case sensitive) option to replace the offending tags ' \
+                       'during processing of the XCCDF ' \
+                       'file to use the ' +
+                       "`$#{offending_tags[0]}` " .colorize(:light_green) +
+                       'syntax in your InSpec profile.'
+          option_two = '(2) Update your XCCDF file to *not use* non-standard XCCDF ' \
+                       'elements within ' +
+                       '`&lt;`,`&gt;`, `<` '.colorize(:red) +
+                       'or '.colorize(:default) +
+                       '`>` '.colorize(:red) +
+                       'as "placeholders", and use something that doesn\'t confuse ' \
+                       'the XML parser, such as : ' +
+                       "`$#{offending_tags[0]}`" .colorize(:light_green)
+          puts option_one
+          puts "\n"
+          puts option_two
+        end
       end
+      HappyMapper::SupportedTypes.register DescriptionDetailsType
     end
-    HappyMapper::SupportedTypes.register DescriptionDetailsType
   end
 end

data/lib/inspec_tools/pdf.rb

@@ -2,9 +2,9 @@ require 'digest'
 
 require_relative '../utilities/inspec_util'
 require_relative '../utilities/extract_pdf_text'
-require_relative '../utilities/extract_nist_cis_mapping'
 require_relative '../utilities/parser'
 require_relative '../utilities/text_cleaner'
+require_relative '../utilities/cis_to_nist'
 
 # rubocop:disable Metrics/AbcSize
 # rubocop:disable Metrics/PerceivedComplexity
@@ -24,7 +24,7 @@ module InspecTools
       @controls = []
       @csv_handle = nil
       @cci_xml = nil
-      @nist_mapping = nil
+      @nist_mapping = Utils::CisToNist.get_mapping('cis_to_nist_critical_controls')
       @pdf_text = ''
       @clean_text = ''
       @transformed_data = ''
@@ -33,7 +33,6 @@ module InspecTools
       @title ||= extract_title
       clean_pdf_text
       transform_data
-      read_excl
       insert_json_metadata
       @profile['controls'] = parse_controls
       @profile['sha256'] = Digest::SHA256.hexdigest @profile.to_s
@@ -122,15 +121,5 @@ module InspecTools
     def write_clean_text
       File.write('debug_text', @clean_text)
     end
-
-    def read_excl
-      nist_map_path = File.join(File.dirname(__FILE__), '../data/NIST_Map_09212017B_CSC-CIS_Critical_Security_Controls_VER_6.1_Excel_9.1.2016.xlsx')
-      excel = Util::ExtractNistMappings.new(nist_map_path)
-      @nist_mapping = excel.full_excl
-    rescue StandardError => e
-      puts "Exception: #{e.message}"
-      puts 'Existing...'
-      exit
-    end
   end
 end

data/lib/inspec_tools/plugin_cli.rb

@@ -200,8 +200,6 @@ module InspecPlugins
       desc 'summary', 'summary parses an inspec results json to create a summary json'
       long_desc InspecTools::Help.text(:summary)
       option :inspec_json, required: true, aliases: '-j'
-      option :output, required: false, aliases: '-o'
-      option :cli, type: :boolean, required: false, aliases: '-c'
       option :verbose, type: :boolean, aliases: '-V'
       option :json_full, type: :boolean, required: false, aliases: '-f'
       option :json_counts, type: :boolean, required: false, aliases: '-k'
@@ -209,7 +207,7 @@ module InspecPlugins
       def summary
         summary = InspecTools::Summary.new(File.read(options[:inspec_json])).to_summary
 
-        if options[:cli]
+        unless options.include?('json_full') || options.include?('json_counts')
           puts "\nOverall compliance: #{summary[:compliance]}%\n\n"
           summary[:status].keys.each do |category|
             puts category
@@ -220,7 +218,6 @@ module InspecPlugins
         end
 
         json_summary = summary.to_json
-        File.write(options[:output], json_summary) if options[:output]
         puts json_summary if options[:json_full]
         puts summary[:status].to_json if options[:json_counts]
       end

data/lib/inspec_tools/summary.rb

@@ -10,8 +10,8 @@ HIGH = 0.7
 MEDIUM = 0.5
 LOW = 0.3
 
-BUCKETS = %w{failed passed no_impact skipped error}.freeze
-TALLYS = %w{total critical high medium low}.freeze
+BUCKETS = %i(failed passed no_impact skipped error).freeze
+TALLYS = %i(total critical high medium low).freeze
 
 THRESHOLD_TEMPLATE = File.expand_path('../data/threshold.yaml', File.dirname(__FILE__))
 
@@ -83,7 +83,7 @@ module InspecTools
         (@summary[:status][:passed][:total]+
          @summary[:status][:failed][:total]+
          @summary[:status][:skipped][:total]+
-         @summary[:status][:error][:total])).round(1)
+         @summary[:status][:error][:total])).floor
     end
 
     def threshold_compliance
@@ -104,13 +104,13 @@ module InspecTools
         TALLYS.each do |tally|
           max = @threshold["#{bucket}.#{tally}.max"]
           min = @threshold["#{bucket}.#{tally}.min"]
-          if max != -1 and status[bucket.to_sym][tally.to_sym] > max
+          if max != -1 and status[bucket][tally] > max
             compliance = false
-            failure << "Expected #{bucket}.#{tally}.max:#{max} got:#{status[bucket.to_sym][tally.to_sym]}"
+            failure << "Expected #{bucket}.#{tally}.max:#{max} got:#{status[bucket][tally]}"
           end
-          if min != -1 and status[bucket.to_sym][tally.to_sym] < min
+          if min != -1 and status[bucket][tally] < min
             compliance = false
-            failure << "Expected #{bucket}.#{tally}.min:#{min} got:#{status[bucket.to_sym][tally.to_sym]}"
+            failure << "Expected #{bucket}.#{tally}.min:#{min} got:#{status[bucket][tally]}"
           end
         end
       end