inspec_tools 2.0.4 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22680d948ef0c9745db5983c3ae8dea966cfa05f2aa2977e2f0409a0d4416a14
-  data.tar.gz: 60954f2699569649f559a4071c97e70a1b0d77eb0f695207106dd4b06887c8b9
+  metadata.gz: 315e07faccf97b313b9493963ef7b9b80ffa7a1459bfa661527c4e827de89676
+  data.tar.gz: e31f0bc2c0006bcb96b9ee46f34c59b8ba71358c2293d7c39874b0fc80b5292b
 SHA512:
-  metadata.gz: 5eab94b7c0f08fe13b37a2c0483e7991c8b625c2134f502e94f8c194d4ef4aee73001d9ccc52686a924ff6d1b4436511706e4ab5cb274476765ebc6e97d42a45
-  data.tar.gz: 07d82a9e11bfb00ee867893cbecbe1ce0d9d7bb0a156ec1e81a80f862ede193c1cb9b4abefcd45bb5d068a7719b9c18a1fbd9053b51e68d9854792b4b90674a5
+  metadata.gz: 50409617c4e6142916f328e5850a5ef3f2d99b9e71671714ce0475708c435242911b6d4455f47fd1f5e50a7778b80e673d4ec74e3ee5d8f66ab5530c42fa8ad9
+  data.tar.gz: e34adf2df0ec1b1bcd5d9224a2621565c5c4efbe79177c2326a3ef8eef6e10922cbc025682c6bade7179add2e59b3b6394419da0cf7c82f1fd8dcfb58efe49f3

data/README.md

@@ -65,12 +65,12 @@ For Docker usage, replace the `inspec_tools` command with the correct Docker com
 
  - **On Linux and Mac**: `docker run -it -v$(pwd):/share mitre/inspec_tools`
  - **On Windows CMD**: `docker run -it -v%cd%:/share mitre/inspec_tools`
- 
+
 Note that all of the above Docker commands will mount your current directory on the Docker container. Ensure that you have navigated to the directory you intend to convert files in before executing the command.
 
 ### generate_map
 
-This command will generate a `mapping.xml` file that can be passed in to the `csv2inspec` command with the `--m` option.
+This command will generate a `mapping.yml` file that can be passed in to the `csv2inspec` command with the `--m` option.
 
 ```
 USAGE: inspec_tools generate_map
@@ -100,13 +100,15 @@ If the specified threshold is not met, an error code (1) is returned along with
 
 The compliance score are rounded down to the nearest whole number. For example a score of 77.3 would be displayed as 77.
 
+Thresholds provided inline (i.e. `-i`) override thresholds provided by files (i.e. `-f`).
+
 ```
 USAGE:  inspec_tools compliance [OPTIONS] -j <inspec-json> -i <threshold-inline>
 	inspec_tools compliance [OPTIONS] -j <inspec-json> -f <threshold-file>
 FLAGS:
 	-j --inspec-json <inspec-json>          : path to InSpec results Json
 	-i --template-inline <threshold-inline> : inline compliance threshold definition
-	-f --template-file <threshold-file>     : yaml file with compliance threshold definition
+	-f --threshold-file <threshold-file>    : yaml file with compliance threshold definition
 Examples:
 
   inspec_tools compliance -j examples/sample_json/rhel-simp.json -i '{compliance.min: 80, failed.critical.max: 0, failed.high.max: 0}'
@@ -135,11 +137,11 @@ failed.high.max: 1
 
 #### In-Line Examples
 ```
-{compliance: {min: 90}, failed: {critical: {max: 0}, high: {max: 0}}}
+"{compliance: {min: 90}, failed: {critical: {max: 0}, high: {max: 0}}}"
 ```
 
 ```
-{compliance.min: 81, failed.critical.max: 0, failed.high.max: 0}
+"{compliance.min: 81, failed.critical.max: 0, failed.high.max: 0}"
 ```
 
 ## summary
@@ -181,16 +183,20 @@ error
 	low : 0
 ```
 
-Using additional flags will override the normal output and only display the output that flag specifies. 
+Using additional flags will override the normal output and only display the output that flag specifies.
+
+USAGE: inspec_tools summary [OPTIONS] -j <inspec-json>
+
+Thresholds provided inline (i.e. `-i`) override thresholds provided by files (i.e. `-t`).
 
-USAGE: inspec_tools summary [OPTIONS] -j <inspec-json> 
 
 ```
 FLAGS:
-  -j --inspec-json <inspec-json>   : path to InSpec results JSON
-  -V --verbose, --no-verbose       : print verbose an debug output
-  -f --json-full, --no-json-full   : print the summary STDOUT as JSON
-  -k --json-counts, --no-json_cou  : print the reslut status to STDOUT as JSON
+  -j --inspec-json <inspec-json>           : path to InSpec results JSON
+  -f --json-full, --no-json-full           : print the summary STDOUT as JSON
+  -k --json-counts, --no-json-counts       : print the result status to STDOUT as JSON
+  -t, --threshold-file=THRESHOLD_FILE]     : path to threshold YAML file
+  -i, --threshold-inline=THRESHOLD_INLINE] : string of text representing threshold YAML inline
 
 Examples:
 
@@ -211,7 +217,7 @@ FLAGS:
 	-f --format [ruby | hash]          : the format you would like (default: ruby) [optional]
 	-s --separate-files [true | false] : output the resulting controls as one or mutiple files (default: true) [optional]
 	-m --metadata <metadata-json>      : path to json file with additional metadata for the inspec.yml file [optional]
-	-r --replace-tags <array>          : A case-sensitive, comma separated list to replace tags with a $ if found in a group rules description tag [optional]
+	-r --replace-tags <array>          : A case-sensitive, space separated list to replace tags with a $ if found in a group rules description tag [optional]
 
 example: inspec_tools xccdf2inspec -x xccdf_file.xml -a attributes.yml -o myprofile -f ruby -s false
 ```
@@ -220,16 +226,18 @@ example: inspec_tools xccdf2inspec -x xccdf_file.xml -a attributes.yml -o myprof
 
 `inspec2xccdf` converts an InSpec profile in json format to a STIG XCCDF Document
 
+See [examples documentation](./examples/inspec2xccdf/README.md) for additional guidance on usage including attribute details.
+
 ```
 USAGE: inspec_tools inspec2xccdf [OPTIONS] -j <inspec-json> -a <xccdf-attr-yml> -o <xccdf-xml>
 
 FLAGS:
 	-j --inspec-json <inspec-json>   : path to InSpec Json file created using command 'inspec json <profile> > example.json'
-	-a --attributes <xccdf-attr-yml> : path to yml file that provides the required attributes for the XCCDF Document. these attributes are parts of XCCDF document which do not fit into the InSpec schema
-	-o --output <xccdf-xml>          : name or path to create the xccdf and title to give the xccdf
-	-V --verbose                     : verbose run [optional]
+	-a --attributes <xccdf-attr-yml> : path to yml file that provides the required attributes for the XCCDF document. These attributes are parts of XCCDF document which do not fit into the InSpec schema.
+	-o --output <xccdf-xml>          : name or path to create the XCCDF and title to give the XCCDF
+        -m, [--metadata=METADATA]        : path to json file with additional host metadata for the XCCDF file
 
-example: inspec_tools inspec2xccdf -j example.json -a attributes.yml -o xccdf.xml
+example: inspec_tools inspec2xccdf -j examples/sample_json/good_nginxresults.json -a lib/data/attributes.yml -o output.xccdf
 ```
 
 ## csv2inspec
@@ -295,7 +303,7 @@ FLAGS:
 	-s --separate-files [true | false] : output the resulting controls as multiple files (default: true) [optional]
 	-d --debug                         : debug run [optional]
 
-example: inspec_tools pdf2inspec -p benchmark.pdf -o /path/to/myprofile -f ruby -s true
+example: inspec_tools pdf2inspec -p examples/CIS_Ubuntu_Linux_16.04_LTS_Benchmark_v1.0.0.pdf -o /path/to/myprofile -f ruby -s true
 ```
 
 ## xlsx2inspec

data/Rakefile

@@ -1,10 +1,9 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
+require 'rake/testtask'
 require File.expand_path('../lib/inspec_tools/version', __FILE__)
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
+  t.libs << 'test'
+  t.libs << 'lib'
   t.test_files = FileList['test/**/*_test.rb']
 end
 
@@ -20,11 +19,86 @@ namespace :test do
       'test/unit/inspec_tools_test.rb'
     ])
   end
+
+  Rake::TestTask.new(:exclude_slow) do |t|
+    t.description = 'Excluding all tests that take more than 3 seconds to complete'
+    t.libs << 'test'
+    t.libs << "lib"
+    t.verbose = true
+    t.test_files = FileList['test/**/*_test.rb'].reject{|file| file.include? 'pdf_test.rb'}.reverse
+  end
+end 
+
+desc 'Build for release'
+task :build_release do
+
+  Rake::Task["generate_mapping_objects"].reenable
+  Rake::Task["generate_mapping_objects"].invoke
+
+  system('gem build inspec_tools.gemspec')
+end
+
+desc 'Generate mapping objects'
+task :generate_mapping_objects do
+  require 'roo'
+
+  nist_mapping_cis_controls = ENV['NIST_MAPPING_CIS_CONTROLS'] || 'NIST_Map_02052020_CIS_Controls_Version_7.1_Implementation_Groups_1.2.xlsx'.freeze
+  nist_mapping_cis_critical_controls = ENV['NIST_MAPPING_CIS_CRITICAL_CONTROLS'] || 'NIST_Map_09212017B_CSC-CIS_Critical_Security_Controls_VER_6.1_Excel_9.1.2016.xlsx'.freeze
+
+  data_root_path = File.join(File.expand_path(__dir__), 'lib', 'data')
+  cis_controls_path = File.join(data_root_path, nist_mapping_cis_controls)
+  cis_critical_controls_path = File.join(data_root_path, nist_mapping_cis_critical_controls)
+
+  raise "#{cis_controls_path} does not exist" unless File.exist?(cis_controls_path)
+
+  raise "#{cis_critical_controls_path} does not exist" unless File.exist?(cis_critical_controls_path)
+
+  marshal_cis_controls(cis_controls_path, data_root_path)
+  marshal_cis_critical_controls(cis_critical_controls_path, data_root_path)
 end
 
-desc 'Build and publish the gem'
-task publish: :build do
-  system("gem push pkg/inspec_tools-#{InspecTools::VERSION}.gem")
+def marshal_cis_controls(cis_controls_path, data_root_path)
+  cis_to_nist = {}
+  Roo::Spreadsheet.open(cis_controls_path).sheet(3).each do |row|
+    if row[3].is_a?(Numeric)
+      cis_to_nist[row[3].to_s] = row[0]
+    else
+      cis_to_nist[row[2].to_s] = row[0] unless (row[2] == '') || row[2].to_i.nil?
+    end
+  end
+  output_file = File.new(File.join(data_root_path, 'cis_to_nist_mapping'), 'w')
+  Marshal.dump(cis_to_nist, output_file)
+  output_file.close
 end
 
-task :default => :test
+def marshal_cis_critical_controls(cis_critical_controls_path, data_root_path)
+  controls_spreadsheet = Roo::Spreadsheet.open(cis_critical_controls_path)
+  controls_spreadsheet.default_sheet = 'VER 6.1 Controls'
+  headings = {}
+  controls_spreadsheet.row(3).each_with_index { |header, idx| headings[header] = idx }
+
+  nist_ver = 4
+  cis_ver = controls_spreadsheet.row(2)[4].split(' ')[-1]
+  control_count = 1
+  mapping = []
+  ((controls_spreadsheet.first_row + 3)..controls_spreadsheet.last_row).each do |row_value|
+    current_row = {}
+    if controls_spreadsheet.row(row_value)[headings['NIST SP 800-53 Control #']].to_s != ''
+      current_row[:nist] = controls_spreadsheet.row(row_value)[headings['NIST SP 800-53 Control #']].to_s
+    else
+      current_row[:nist] = 'Not Mapped'
+    end
+    current_row[:nist_ver] = nist_ver
+    if controls_spreadsheet.row(row_value)[headings['Control']].to_s == ''
+      current_row[:cis] = control_count.to_s
+      control_count += 1
+    else
+      current_row[:cis] = controls_spreadsheet.row(row_value)[headings['Control']].to_s
+    end
+    current_row[:cis_ver] = cis_ver
+    mapping << current_row
+  end
+  output_file = File.new(File.join(data_root_path, 'cis_to_nist_critical_controls'), 'w')
+  Marshal.dump(mapping, output_file)
+  output_file.close
+end

data/lib/happy_mapper_tools/benchmark.rb

@@ -99,6 +99,88 @@ module HappyMapperTools
       element :content, String, tag: 'check-content'
     end
 
+    class MessageType
+      include HappyMapper
+      attribute :severity, String, tag: 'severity'
+      content :message, String
+    end
+
+    class RuleResultType
+      include HappyMapper
+      attribute :idref, String, tag: 'idref'
+      attribute :severity, String, tag: 'severity'
+      attribute :time, String, tag: 'time'
+      attribute :weight, String, tag: 'weight'
+      element :result, String, tag: 'result'
+      # element override - Not implemented. Does not apply to Inspec execution
+      has_many :ident, Ident, tag: 'ident'
+      # Note: element metadata not implemented at this time
+      has_many :message, MessageType, tag: 'message'
+      has_many :instance, String, tag: 'instance'
+      element :fix, Fix, tag: 'fix'
+      element :check, Check, tag: 'check'
+    end
+
+    class ScoreType
+      include HappyMapper
+
+      def initialize(system, maximum, score)
+        @system = system
+        @maximum = maximum
+        @score = score
+      end
+
+      attribute :system, String, tag: 'system'
+      attribute :maximum, String, tag: 'maximum' # optional attribute
+      content :score, String
+    end
+
+    class CPE2idrefType
+      include HappyMapper
+      attribute :idref, String, tag: 'idref'
+    end
+
+    class IdentityType
+      include HappyMapper
+      attribute :authenticated, Boolean, tag: 'authenticated'
+      attribute :privileged, Boolean, tag: 'privileged'
+      content :identity, String
+    end
+
+    class Fact
+      include HappyMapper
+      attribute :name, String, tag: 'name'
+      attribute :type, String, tag: 'type'
+      content :fact, String
+    end
+
+    class TargetFact
+      include HappyMapper
+      has_many :fact, Fact, tag: 'fact'
+    end
+
+    class TestResult
+      include HappyMapper
+      # Note: element benchmark not implemented at this time since this is same file
+      # Note: element title not implemented due to no mapping from Chef Inspec
+      element :remark, String, tag: 'remark'
+      has_many :organization, String, tag: 'organization'
+      element :identity, IdentityType, tag: 'identity'
+      element :target, String, tag: 'target'
+      has_many :target_address, String, tag: 'target-address'
+      element :target_facts, TargetFact, tag: 'target-facts'
+      element :platform, CPE2idrefType, tag: 'platform'
+      # Note: element profile not implemented since Benchmark profile is also not implemented
+      has_many :rule_result, RuleResultType, tag: 'rule-result'
+      has_many :score, ScoreType, tag: 'score' # One minimum
+      # Note: element signature not implemented due to no mapping from Chef Inspec
+      attribute :id, String, tag: 'id'
+      attribute :starttime, String, tag: 'start-time'
+      attribute :endtime, String, tag: 'end-time'
+      # Note: attribute test-system not implemented at this time due to unknown CPE value for Chef Inspec
+      attribute :version, String, tag: 'version'
+    end
+
     # Class Profile maps from the 'Profile' from Benchmark XML file using HappyMapper
     class Profile
       include HappyMapper
@@ -156,6 +238,7 @@ module HappyMapperTools
       element :version, String, tag: 'version'
       has_many :profile, Profile, tag: 'Profile'
       has_many :group, Group, tag: 'Group'
+      element :testresult, TestResult, tag: 'TestResult'
     end
   end
 end

data/lib/happy_mapper_tools/stig_attributes.rb

@@ -38,6 +38,7 @@ module HappyMapperTools
       element :documentable, Boolean, tag: 'Documentable'
       element :mitigations, String, tag: 'Mitigations'
       element :severity_override_guidance, String, tag: 'SeverityOverrideGuidance'
+      element :security_override_guidance, String, tag: 'SecurityOverrideGuidance'
       element :potential_impacts, String, tag: 'PotentialImpacts'
       element :third_party_tools, String, tag: 'ThirdPartyTools'
       element :mitigation_controls, String, tag: 'MitigationControl'
@@ -53,7 +54,8 @@ module HappyMapperTools
 
       detail_tags = %i(vuln_discussion false_positives false_negatives documentable
                        mitigations severity_override_guidance potential_impacts
-                       third_party_tools mitigation_controls responsibility ia_controls)
+                       third_party_tools mitigation_controls responsibility ia_controls
+                       security_override_guidance)
 
       detail_tags.each do |name|
         define_method name do
@@ -140,57 +142,75 @@ module HappyMapperTools
     end
 
     class DescriptionDetailsType
-      def self.type
-        DescriptionDetails
-      end
+      class << self
+        def type
+          DescriptionDetails
+        end
 
-      def self.apply(value) # rubocop:disable Metrics/AbcSize
-        value = value.gsub('&', 'and')
-        DescriptionDetails.parse "<Details>#{value}</Details>"
-      rescue Nokogiri::XML::SyntaxError
-        allowed_tags = %w{VulnDiscussion FalsePositives FalseNegatives Documentable
-                          Mitigations SeverityOverrideGuidance PotentialImpacts
-                          PotentialImpacts ThirdPartyTools MitigationControl
-                          Responsibility IAControls}
-
-        tags_found = value.scan(%r{(?<=<)([^\/]*?)((?= \/>)|(?=>))}).to_a
-
-        tags_found = tags_found.uniq.flatten.reject!(&:empty?)
-        offending_tags = tags_found - allowed_tags
-
-        if offending_tags.count > 1
-          puts "\n\nThe non-standard tags: #{offending_tags.to_s.colorize(:red)}" \
-               ' were found in: ' + "\n\n#{value}"
-        else
-          puts "\n\nThe non-standard tag: #{offending_tags.to_s.colorize(:red)}" \
-               ' was found in: ' + "\n\n#{value}"
+        def apply(value)
+          value = value.gsub('&', 'and')
+          DescriptionDetails.parse "<Details>#{value}</Details>"
+        rescue Nokogiri::XML::SyntaxError => e
+          if e.to_s.include?('StartTag')
+            report_invalid_start_tag(value, e)
+          else
+            report_disallowed_tags(value)
+          end
+        end
+
+        def apply?(value, _convert_to_type)
+          value.is_a?(String)
         end
-        puts "\n\nPlease:\n "
-        option_one = '(1) ' + '(best)'.colorize(:green) + ' Use the ' +
-                     '`-r --replace-tags array` '.colorize(:light_yellow) +
-                     '(case sensitive) option to replace the offending tags ' \
-                     'during processing of the XCCDF ' \
-                     'file to use the ' +
-                     "`$#{offending_tags[0]}` " .colorize(:light_green) +
-                     'syntax in your InSpec profile.'
-        option_two = '(2) Update your XCCDF file to *not use* non-standard XCCDF ' \
-                     'elements within ' +
-                     '`&lt;`,`&gt;`, `<` '.colorize(:red) +
-                     'or '.colorize(:default) +
-                     '`>` '.colorize(:red) +
-                     'as "placeholders", and use something that doesn\'t confuse ' \
-                     'the XML parser, such as : ' +
-                     "`$#{offending_tags[0]}`" .colorize(:light_green)
-        puts option_one
-        puts "\n"
-        puts option_two
-        # exit
-      end
 
-      def self.apply?(value, _convert_to_type)
-        value.is_a?(String)
+        private
+
+        def report_invalid_start_tag(value, error)
+          puts error.to_s.colorize(:red)
+          column = error.column - '<Details>'.length - 2
+          puts "Error around #{value[column-10..column+10].colorize(:light_yellow)}"
+          exit(1)
+        end
+
+        def report_disallowed_tags(value)
+          allowed_tags = %w{VulnDiscussion FalsePositives FalseNegatives Documentable
+                            Mitigations SeverityOverrideGuidance PotentialImpacts
+                            PotentialImpacts ThirdPartyTools MitigationControl
+                            Responsibility IAControl SecurityOverrideGuidance}
+
+          tags_found = value.scan(%r{(?<=<)([^\/]*?)((?= \/>)|(?=>))}).to_a
+
+          tags_found = tags_found.uniq.flatten.reject!(&:empty?)
+          offending_tags = tags_found - allowed_tags
+
+          if offending_tags.count > 1
+            puts "\n\nThe non-standard tags: #{offending_tags.to_s.colorize(:red)}" \
+                 ' were found in: ' + "\n\n#{value}"
+          else
+            puts "\n\nThe non-standard tag: #{offending_tags.to_s.colorize(:red)}" \
+                 ' was found in: ' + "\n\n#{value}"
+          end
+          puts "\n\nPlease:\n "
+          option_one = '(1) ' + '(best)'.colorize(:green) + ' Use the ' +
+                       '`-r --replace-tags array` '.colorize(:light_yellow) +
+                       '(case sensitive) option to replace the offending tags ' \
+                       'during processing of the XCCDF ' \
+                       'file to use the ' +
+                       "`$#{offending_tags[0]}` " .colorize(:light_green) +
+                       'syntax in your InSpec profile.'
+          option_two = '(2) Update your XCCDF file to *not use* non-standard XCCDF ' \
+                       'elements within ' +
+                       '`&lt;`,`&gt;`, `<` '.colorize(:red) +
+                       'or '.colorize(:default) +
+                       '`>` '.colorize(:red) +
+                       'as "placeholders", and use something that doesn\'t confuse ' \
+                       'the XML parser, such as : ' +
+                       "`$#{offending_tags[0]}`" .colorize(:light_green)
+          puts option_one
+          puts "\n"
+          puts option_two
+        end
       end
+      HappyMapper::SupportedTypes.register DescriptionDetailsType
     end
-    HappyMapper::SupportedTypes.register DescriptionDetailsType
   end
 end