inspec_tools 2.0.3 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d99887f62c18c23143d73ceb049277beaf493a765dc1df5beec9fee9b9bb7dca
-  data.tar.gz: c2613ab9b76c9dae510ba8c57a27664989161cc8dd0cfec099f1365a9743030f
+  metadata.gz: eee290c08f68bf8bb7894537a1871e0bdf5144266ee50932ea70af3018d5f94e
+  data.tar.gz: 42ed4ee43680aaceb6fa12c03bfe7c29dd88a33399787b7053510716182c627e
 SHA512:
-  metadata.gz: e8f1fd2c3b491e7c3ef65c76c250e16e6e00cbb03ae0d3ed8c6c23bc8598ba31f8315bd72e6bcfb76b22c4a054f881852de0f290789c6d5bab304afc616003f4
-  data.tar.gz: ad843429c15e5e10c655860a6178b20d088decbd4f6bc92817183635b45c7478be16b30baf6883d3e1565f47c0926226e117482351137811829c9e147de4be3e
+  metadata.gz: 43e9b4038ff40f6aee88a16fb63e536b7d3918f15b8a3f9b3fcf87688eb1a167cfd08a7f45cac6fcdd68a2cdd1b5f1ac9ab9a85a2bd42c983b9e963c0b18d96d
+  data.tar.gz: cd750538af91a05636ad406b80f1adc83bbacfa382143d472bbced9265227f3da245a8a23dd7de0af3636c1650b1c563c6621dc48f524db1550518e7bcc55ae5

data/README.md

@@ -65,12 +65,12 @@ For Docker usage, replace the `inspec_tools` command with the correct Docker com
 
  - **On Linux and Mac**: `docker run -it -v$(pwd):/share mitre/inspec_tools`
  - **On Windows CMD**: `docker run -it -v%cd%:/share mitre/inspec_tools`
- 
+
 Note that all of the above Docker commands will mount your current directory on the Docker container. Ensure that you have navigated to the directory you intend to convert files in before executing the command.
 
 ### generate_map
 
-This command will generate a `mapping.xml` file that can be passed in to the `csv2inspec` command with the `--m` option.
+This command will generate a `mapping.yml` file that can be passed in to the `csv2inspec` command with the `--m` option.
 
 ```
 USAGE: inspec_tools generate_map
@@ -181,9 +181,9 @@ error
 	low : 0
 ```
 
-Using additional flags will override the normal output and only display the output that flag specifies. 
+Using additional flags will override the normal output and only display the output that flag specifies.
 
-USAGE: inspec_tools summary [OPTIONS] -j <inspec-json> 
+USAGE: inspec_tools summary [OPTIONS] -j <inspec-json>
 
 ```
 FLAGS:
@@ -211,7 +211,7 @@ FLAGS:
 	-f --format [ruby | hash]          : the format you would like (default: ruby) [optional]
 	-s --separate-files [true | false] : output the resulting controls as one or mutiple files (default: true) [optional]
 	-m --metadata <metadata-json>      : path to json file with additional metadata for the inspec.yml file [optional]
-	-r --replace-tags <array>          : A case-sensitive, comma separated list to replace tags with a $ if found in a group rules description tag [optional]
+	-r --replace-tags <array>          : A case-sensitive, space separated list to replace tags with a $ if found in a group rules description tag [optional]
 
 example: inspec_tools xccdf2inspec -x xccdf_file.xml -a attributes.yml -o myprofile -f ruby -s false
 ```
@@ -220,16 +220,18 @@ example: inspec_tools xccdf2inspec -x xccdf_file.xml -a attributes.yml -o myprof
 
 `inspec2xccdf` converts an InSpec profile in json format to a STIG XCCDF Document
 
+See [examples documentation](./examples/inspec2xccdf/README.md) for additional guidance on usage including attribute details.
+
 ```
 USAGE: inspec_tools inspec2xccdf [OPTIONS] -j <inspec-json> -a <xccdf-attr-yml> -o <xccdf-xml>
 
 FLAGS:
 	-j --inspec-json <inspec-json>   : path to InSpec Json file created using command 'inspec json <profile> > example.json'
-	-a --attributes <xccdf-attr-yml> : path to yml file that provides the required attributes for the XCCDF Document. these attributes are parts of XCCDF document which do not fit into the InSpec schema
-	-o --output <xccdf-xml>          : name or path to create the xccdf and title to give the xccdf
-	-V --verbose                     : verbose run [optional]
+	-a --attributes <xccdf-attr-yml> : path to yml file that provides the required attributes for the XCCDF document. These attributes are parts of XCCDF document which do not fit into the InSpec schema.
+	-o --output <xccdf-xml>          : name or path to create the XCCDF and title to give the XCCDF
+        -m, [--metadata=METADATA]        : path to json file with additional host metadata for the XCCDF file
 
-example: inspec_tools inspec2xccdf -j example.json -a attributes.yml -o xccdf.xml
+example: inspec_tools inspec2xccdf -j examples/sample_json/good_nginxresults.json -a lib/data/attributes.yml -o output.xccdf
 ```
 
 ## csv2inspec
@@ -295,7 +297,7 @@ FLAGS:
 	-s --separate-files [true | false] : output the resulting controls as multiple files (default: true) [optional]
 	-d --debug                         : debug run [optional]
 
-example: inspec_tools pdf2inspec -p benchmark.pdf -o /path/to/myprofile -f ruby -s true
+example: inspec_tools pdf2inspec -p examples/CIS_Ubuntu_Linux_16.04_LTS_Benchmark_v1.0.0.pdf -o /path/to/myprofile -f ruby -s true
 ```
 
 ## xlsx2inspec

data/Rakefile

@@ -1,10 +1,9 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
+require 'rake/testtask'
 require File.expand_path('../lib/inspec_tools/version', __FILE__)
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
+  t.libs << 'test'
+  t.libs << 'lib'
   t.test_files = FileList['test/**/*_test.rb']
 end
 
@@ -20,11 +19,86 @@ namespace :test do
       'test/unit/inspec_tools_test.rb'
     ])
   end
+
+  Rake::TestTask.new(:exclude_slow) do |t|
+    t.description = 'Excluding all tests that take more than 3 seconds to complete'
+    t.libs << 'test'
+    t.libs << "lib"
+    t.verbose = true
+    t.test_files = FileList['test/**/*_test.rb'].reject{|file| file.include? 'pdf_test.rb'}.reverse
+  end
+end 
+
+desc 'Build for release'
+task :build_release do
+
+  Rake::Task["generate_mapping_objects"].reenable
+  Rake::Task["generate_mapping_objects"].invoke
+
+  system('gem build inspec_tools.gemspec')
+end
+
+desc 'Generate mapping objects'
+task :generate_mapping_objects do
+  require 'roo'
+
+  nist_mapping_cis_controls = ENV['NIST_MAPPING_CIS_CONTROLS'] || 'NIST_Map_02052020_CIS_Controls_Version_7.1_Implementation_Groups_1.2.xlsx'.freeze
+  nist_mapping_cis_critical_controls = ENV['NIST_MAPPING_CIS_CRITICAL_CONTROLS'] || 'NIST_Map_09212017B_CSC-CIS_Critical_Security_Controls_VER_6.1_Excel_9.1.2016.xlsx'.freeze
+
+  data_root_path = File.join(File.expand_path(__dir__), 'lib', 'data')
+  cis_controls_path = File.join(data_root_path, nist_mapping_cis_controls)
+  cis_critical_controls_path = File.join(data_root_path, nist_mapping_cis_critical_controls)
+
+  raise "#{cis_controls_path} does not exist" unless File.exist?(cis_controls_path)
+
+  raise "#{cis_critical_controls_path} does not exist" unless File.exist?(cis_critical_controls_path)
+
+  marshal_cis_controls(cis_controls_path, data_root_path)
+  marshal_cis_critical_controls(cis_critical_controls_path, data_root_path)
 end
 
-desc 'Build and publish the gem'
-task publish: :build do
-  system("gem push pkg/inspec_tools-#{InspecTools::VERSION}.gem")
+def marshal_cis_controls(cis_controls_path, data_root_path)
+  cis_to_nist = {}
+  Roo::Spreadsheet.open(cis_controls_path).sheet(3).each do |row|
+    if row[3].is_a?(Numeric)
+      cis_to_nist[row[3].to_s] = row[0]
+    else
+      cis_to_nist[row[2].to_s] = row[0] unless (row[2] == '') || row[2].to_i.nil?
+    end
+  end
+  output_file = File.new(File.join(data_root_path, 'cis_to_nist_mapping'), 'w')
+  Marshal.dump(cis_to_nist, output_file)
+  output_file.close
 end
 
-task :default => :test
+def marshal_cis_critical_controls(cis_critical_controls_path, data_root_path)
+  controls_spreadsheet = Roo::Spreadsheet.open(cis_critical_controls_path)
+  controls_spreadsheet.default_sheet = 'VER 6.1 Controls'
+  headings = {}
+  controls_spreadsheet.row(3).each_with_index { |header, idx| headings[header] = idx }
+
+  nist_ver = 4
+  cis_ver = controls_spreadsheet.row(2)[4].split(' ')[-1]
+  control_count = 1
+  mapping = []
+  ((controls_spreadsheet.first_row + 3)..controls_spreadsheet.last_row).each do |row_value|
+    current_row = {}
+    if controls_spreadsheet.row(row_value)[headings['NIST SP 800-53 Control #']].to_s != ''
+      current_row[:nist] = controls_spreadsheet.row(row_value)[headings['NIST SP 800-53 Control #']].to_s
+    else
+      current_row[:nist] = 'Not Mapped'
+    end
+    current_row[:nist_ver] = nist_ver
+    if controls_spreadsheet.row(row_value)[headings['Control']].to_s == ''
+      current_row[:cis] = control_count.to_s
+      control_count += 1
+    else
+      current_row[:cis] = controls_spreadsheet.row(row_value)[headings['Control']].to_s
+    end
+    current_row[:cis_ver] = cis_ver
+    mapping << current_row
+  end
+  output_file = File.new(File.join(data_root_path, 'cis_to_nist_critical_controls'), 'w')
+  Marshal.dump(mapping, output_file)
+  output_file.close
+end

data/lib/data/rubocop.yml

@@ -0,0 +1,4 @@
+AllCops:
+  DisabledByDefault: true
+Style/StringLiterals:
+  Enabled: true

data/lib/happy_mapper_tools/benchmark.rb

@@ -99,6 +99,88 @@ module HappyMapperTools
       element :content, String, tag: 'check-content'
     end
 
+    class MessageType
+      include HappyMapper
+      attribute :severity, String, tag: 'severity'
+      content :message, String
+    end
+
+    class RuleResultType
+      include HappyMapper
+      attribute :idref, String, tag: 'idref'
+      attribute :severity, String, tag: 'severity'
+      attribute :time, String, tag: 'time'
+      attribute :weight, String, tag: 'weight'
+      element :result, String, tag: 'result'
+      # element override - Not implemented. Does not apply to Inspec execution
+      has_many :ident, Ident, tag: 'ident'
+      # Note: element metadata not implemented at this time
+      has_many :message, MessageType, tag: 'message'
+      has_many :instance, String, tag: 'instance'
+      element :fix, Fix, tag: 'fix'
+      element :check, Check, tag: 'check'
+    end
+
+    class ScoreType
+      include HappyMapper
+
+      def initialize(system, maximum, score)
+        @system = system
+        @maximum = maximum
+        @score = score
+      end
+
+      attribute :system, String, tag: 'system'
+      attribute :maximum, String, tag: 'maximum' # optional attribute
+      content :score, String
+    end
+
+    class CPE2idrefType
+      include HappyMapper
+      attribute :idref, String, tag: 'idref'
+    end
+
+    class IdentityType
+      include HappyMapper
+      attribute :authenticated, Boolean, tag: 'authenticated'
+      attribute :privileged, Boolean, tag: 'privileged'
+      content :identity, String
+    end
+
+    class Fact
+      include HappyMapper
+      attribute :name, String, tag: 'name'
+      attribute :type, String, tag: 'type'
+      content :fact, String
+    end
+
+    class TargetFact
+      include HappyMapper
+      has_many :fact, Fact, tag: 'fact'
+    end
+
+    class TestResult
+      include HappyMapper
+      # Note: element benchmark not implemented at this time since this is same file
+      # Note: element title not implemented due to no mapping from Chef Inspec
+      element :remark, String, tag: 'remark'
+      has_many :organization, String, tag: 'organization'
+      element :identity, IdentityType, tag: 'identity'
+      element :target, String, tag: 'target'
+      has_many :target_address, String, tag: 'target-address'
+      element :target_facts, TargetFact, tag: 'target-facts'
+      element :platform, CPE2idrefType, tag: 'platform'
+      # Note: element profile not implemented since Benchmark profile is also not implemented
+      has_many :rule_result, RuleResultType, tag: 'rule-result'
+      has_many :score, ScoreType, tag: 'score' # One minimum
+      # Note: element signature not implemented due to no mapping from Chef Inspec
+      attribute :id, String, tag: 'id'
+      attribute :starttime, String, tag: 'start-time'
+      attribute :endtime, String, tag: 'end-time'
+      # Note: attribute test-system not implemented at this time due to unknown CPE value for Chef Inspec
+      attribute :version, String, tag: 'version'
+    end
+
     # Class Profile maps from the 'Profile' from Benchmark XML file using HappyMapper
     class Profile
       include HappyMapper
@@ -156,6 +238,7 @@ module HappyMapperTools
       element :version, String, tag: 'version'
       has_many :profile, Profile, tag: 'Profile'
       has_many :group, Group, tag: 'Group'
+      element :testresult, TestResult, tag: 'TestResult'
     end
   end
 end

data/lib/happy_mapper_tools/stig_attributes.rb

@@ -38,6 +38,7 @@ module HappyMapperTools
       element :documentable, Boolean, tag: 'Documentable'
       element :mitigations, String, tag: 'Mitigations'
       element :severity_override_guidance, String, tag: 'SeverityOverrideGuidance'
+      element :security_override_guidance, String, tag: 'SecurityOverrideGuidance'
       element :potential_impacts, String, tag: 'PotentialImpacts'
       element :third_party_tools, String, tag: 'ThirdPartyTools'
       element :mitigation_controls, String, tag: 'MitigationControl'
@@ -53,7 +54,8 @@ module HappyMapperTools
 
       detail_tags = %i(vuln_discussion false_positives false_negatives documentable
                        mitigations severity_override_guidance potential_impacts
-                       third_party_tools mitigation_controls responsibility ia_controls)
+                       third_party_tools mitigation_controls responsibility ia_controls
+                       security_override_guidance)
 
       detail_tags.each do |name|
         define_method name do
@@ -140,57 +142,75 @@ module HappyMapperTools
     end
 
     class DescriptionDetailsType
-      def self.type
-        DescriptionDetails
-      end
+      class << self
+        def type
+          DescriptionDetails
+        end
 
-      def self.apply(value) # rubocop:disable Metrics/AbcSize
-        value = value.gsub('&', 'and')
-        DescriptionDetails.parse "<Details>#{value}</Details>"
-      rescue Nokogiri::XML::SyntaxError
-        allowed_tags = %w{VulnDiscussion FalsePositives FalseNegatives Documentable
-                          Mitigations SeverityOverrideGuidance PotentialImpacts
-                          PotentialImpacts ThirdPartyTools MitigationControl
-                          Responsibility IAControls}
-
-        tags_found = value.scan(%r{(?<=<)([^\/]*?)((?= \/>)|(?=>))}).to_a
-
-        tags_found = tags_found.uniq.flatten.reject!(&:empty?)
-        offending_tags = tags_found - allowed_tags
-
-        if offending_tags.count > 1
-          puts "\n\nThe non-standard tags: #{offending_tags.to_s.colorize(:red)}" \
-               ' were found in: ' + "\n\n#{value}"
-        else
-          puts "\n\nThe non-standard tag: #{offending_tags.to_s.colorize(:red)}" \
-               ' was found in: ' + "\n\n#{value}"
+        def apply(value)
+          value = value.gsub('&', 'and')
+          DescriptionDetails.parse "<Details>#{value}</Details>"
+        rescue Nokogiri::XML::SyntaxError => e
+          if e.to_s.include?('StartTag')
+            report_invalid_start_tag(value, e)
+          else
+            report_disallowed_tags(value)
+          end
+        end
+
+        def apply?(value, _convert_to_type)
+          value.is_a?(String)
         end
-        puts "\n\nPlease:\n "
-        option_one = '(1) ' + '(best)'.colorize(:green) + ' Use the ' +
-                     '`-r --replace-tags array` '.colorize(:light_yellow) +
-                     '(case sensitive) option to replace the offending tags ' \
-                     'during processing of the XCCDF ' \
-                     'file to use the ' +
-                     "`$#{offending_tags[0]}` " .colorize(:light_green) +
-                     'syntax in your InSpec profile.'
-        option_two = '(2) Update your XCCDF file to *not use* non-standard XCCDF ' \
-                     'elements within ' +
-                     '`&lt;`,`&gt;`, `<` '.colorize(:red) +
-                     'or '.colorize(:default) +
-                     '`>` '.colorize(:red) +
-                     'as "placeholders", and use something that doesn\'t confuse ' \
-                     'the XML parser, such as : ' +
-                     "`$#{offending_tags[0]}`" .colorize(:light_green)
-        puts option_one
-        puts "\n"
-        puts option_two
-        # exit
-      end
 
-      def self.apply?(value, _convert_to_type)
-        value.is_a?(String)
+        private
+
+        def report_invalid_start_tag(value, error)
+          puts error.to_s.colorize(:red)
+          column = error.column - '<Details>'.length - 2
+          puts "Error around #{value[column-10..column+10].colorize(:light_yellow)}"
+          exit(1)
+        end
+
+        def report_disallowed_tags(value)
+          allowed_tags = %w{VulnDiscussion FalsePositives FalseNegatives Documentable
+                            Mitigations SeverityOverrideGuidance PotentialImpacts
+                            PotentialImpacts ThirdPartyTools MitigationControl
+                            Responsibility IAControl SecurityOverrideGuidance}
+
+          tags_found = value.scan(%r{(?<=<)([^\/]*?)((?= \/>)|(?=>))}).to_a
+
+          tags_found = tags_found.uniq.flatten.reject!(&:empty?)
+          offending_tags = tags_found - allowed_tags
+
+          if offending_tags.count > 1
+            puts "\n\nThe non-standard tags: #{offending_tags.to_s.colorize(:red)}" \
+                 ' were found in: ' + "\n\n#{value}"
+          else
+            puts "\n\nThe non-standard tag: #{offending_tags.to_s.colorize(:red)}" \
+                 ' was found in: ' + "\n\n#{value}"
+          end
+          puts "\n\nPlease:\n "
+          option_one = '(1) ' + '(best)'.colorize(:green) + ' Use the ' +
+                       '`-r --replace-tags array` '.colorize(:light_yellow) +
+                       '(case sensitive) option to replace the offending tags ' \
+                       'during processing of the XCCDF ' \
+                       'file to use the ' +
+                       "`$#{offending_tags[0]}` " .colorize(:light_green) +
+                       'syntax in your InSpec profile.'
+          option_two = '(2) Update your XCCDF file to *not use* non-standard XCCDF ' \
+                       'elements within ' +
+                       '`&lt;`,`&gt;`, `<` '.colorize(:red) +
+                       'or '.colorize(:default) +
+                       '`>` '.colorize(:red) +
+                       'as "placeholders", and use something that doesn\'t confuse ' \
+                       'the XML parser, such as : ' +
+                       "`$#{offending_tags[0]}`" .colorize(:light_green)
+          puts option_one
+          puts "\n"
+          puts option_two
+        end
       end
+      HappyMapper::SupportedTypes.register DescriptionDetailsType
     end
-    HappyMapper::SupportedTypes.register DescriptionDetailsType
   end
 end

data/lib/inspec_tools/csv.rb

@@ -1,10 +1,10 @@
 require 'csv'
-require 'nokogiri'
-require 'word_wrap'
 require 'yaml'
 require 'digest'
 
 require_relative '../utilities/inspec_util'
+require_relative '../utilities/cci_xml'
+require_relative '../utilities/mapping_validator'
 
 # rubocop:disable Metrics/AbcSize
 # rubocop:disable Metrics/PerceivedComplexity
@@ -16,34 +16,25 @@ module InspecTools
     def initialize(csv, mapping, name, verbose = false)
       @name = name
       @csv = csv
-      @mapping = mapping
+      @mapping = Utils::MappingValidator.validate(mapping)
       @verbose = verbose
       @csv.shift if @mapping['skip_csv_header']
     end
 
-    def to_ckl
-      # TODO
-    end
-
-    def to_xccdf
-      # TODO
-    end
-
     def to_inspec
       @controls = []
-      @cci_xml = nil
       @profile = {}
-      read_cci_xml
-      insert_json_metadata
+      @cci_xml = Utils::CciXml.get_cci_list('U_CCI_List.xml')
+      insert_metadata
       parse_controls
       @profile['controls'] = @controls
-      @profile['sha256'] = Digest::SHA256.hexdigest @profile.to_s
+      @profile['sha256'] = Digest::SHA256.hexdigest(@profile.to_s)
       @profile
     end
 
     private
 
-    def insert_json_metadata
+    def insert_metadata
       @profile['name'] = @name
       @profile['title'] = 'InSpec Profile'
       @profile['maintainer'] = 'The Authors'
@@ -60,35 +51,37 @@ module InspecTools
       }
     end
 
-    def read_cci_xml
-      cci_list_path = File.join(File.dirname(__FILE__), '../data/U_CCI_List.xml')
-      @cci_xml = Nokogiri::XML(File.open(cci_list_path))
-      @cci_xml.remove_namespaces!
-    rescue StandardError => e
-      puts "Exception: #{e.message}"
-    end
-
     def get_nist_reference(cci_number)
       item_node = @cci_xml.xpath("//cci_list/cci_items/cci_item[@id='#{cci_number}']")[0] unless @cci_xml.nil?
-      unless item_node.nil?
-        nist_ref = item_node.xpath('./references/reference[not(@version <= preceding-sibling::reference/@version) and not(@version <=following-sibling::reference/@version)]/@index').text
-        nist_ver = item_node.xpath('./references/reference[not(@version <= preceding-sibling::reference/@version) and not(@version <=following-sibling::reference/@version)]/@version').text
-      end
-      [nist_ref, nist_ver]
+      return nil if item_node.nil?
+
+      [] << item_node.xpath('./references/reference[not(@version <= preceding-sibling::reference/@version) and not(@version <=following-sibling::reference/@version)]/@index').text
+    end
+
+    def get_cci_number(cell)
+      # Return nil if a mapping to the CCI was not provided or if there is not content in the CSV cell.
+      return nil if cell.nil? || @mapping['control.tags']['cci'].nil?
+
+      # If the content has been exported from STIG Viewer, the cell will have extra information
+      cell.split("\n").first
     end
 
     def parse_controls
       @csv.each do |row|
-        print '.'
         control = {}
         control['id']     = row[@mapping['control.id']]     unless @mapping['control.id'].nil? || row[@mapping['control.id']].nil?
         control['title']  = row[@mapping['control.title']]  unless @mapping['control.title'].nil? || row[@mapping['control.title']].nil?
         control['desc']   = row[@mapping['control.desc']]   unless @mapping['control.desc'].nil? || row[@mapping['control.desc']].nil?
         control['tags'] = {}
-        nist, nist_rev = get_nist_reference(row[@mapping['control.tags']['cci']]) unless @mapping['control.tags']['cci'].nil? || row[@mapping['control.tags']['cci']].nil?
-        control['tags']['nist'] = [nist, 'Rev_' + nist_rev] unless nist.nil? || nist_rev.nil?
+        cci_number = get_cci_number(row[@mapping['control.tags']['cci']])
+        nist = get_nist_reference(cci_number) unless cci_number.nil?
+        control['tags']['nist'] = nist unless nist.nil? || nist.include?(nil)
         @mapping['control.tags'].each do |tag|
-          control['tags'][tag.first.to_s] = row[tag.last] unless row[tag.last].nil?
+          if tag.first == 'cci'
+            control['tags'][tag.first] = cci_number
+            next
+          end
+          control['tags'][tag.first] = row[tag.last] unless row[tag.last].nil?
         end
         unless @mapping['control.tags']['severity'].nil? || row[@mapping['control.tags']['severity']].nil?
           control['impact'] = Utils::InspecUtil.get_impact(row[@mapping['control.tags']['severity']])