inspec_tools 2.0.6 → 2.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43b88686ec67ec39b204d239fc5374c6448b9cfa1c1bd6b7832966b90619cc23
-  data.tar.gz: dca55a3609c9ff90186d7e83f6017b4653b211b871e4ff2671f834e203a39879
+  metadata.gz: f1facda0eab24bdef7fafd9c8c92c5c94efedee3465bbf589b98fd4a61011be2
+  data.tar.gz: a38564b26ff610046007e427c8b45950b743087703224470b0513e94941d6cc7
 SHA512:
-  metadata.gz: eab9120d563910e628f1dbfa47dc11b1abfff5992a040a23ad7978ae42567584a34dcc8bbfa0a926b3abfb55ed58f04f2f91a056f458ee75f9ca353745c3ebb5
-  data.tar.gz: 17c1fea7003f96df5ce83fbda87165144b389ae6475521bb4e14af8eda259f2093f7a206ab7f9537994b645595fdcb4fbb3c3ece81aa283c4705e3c5d3553307
+  metadata.gz: 45b1990c5f4a2e5fa1fd21af344ac9cd9cdf5b60fce134ed80e7e4593a2c315ad09e994d5efc711fa0d6485f26899a5c35dec5e8ba0c37686cfaca9f483304da
+  data.tar.gz: 1022c5eae8dd7e6645e39fae3fbcb4babc195c2876d76d5f6496bb976a7515cb004d476176e19d70bba0507f8c8f1c355f48899c00fb39e79adcc2f26333fc08

data/README.md

@@ -70,7 +70,7 @@ Note that all of the above Docker commands will mount your current directory on
 
 ### generate_map
 
-This command will generate a `mapping.xml` file that can be passed in to the `csv2inspec` command with the `--m` option.
+This command will generate a `mapping.yml` file that can be passed in to the `csv2inspec` command with the `--m` option.
 
 ```
 USAGE: inspec_tools generate_map
@@ -100,13 +100,15 @@ If the specified threshold is not met, an error code (1) is returned along with
 
 The compliance score are rounded down to the nearest whole number. For example a score of 77.3 would be displayed as 77.
 
+Thresholds provided inline (i.e. `-i`) override thresholds provided by files (i.e. `-f`).
+
 ```
 USAGE:  inspec_tools compliance [OPTIONS] -j <inspec-json> -i <threshold-inline>
 	inspec_tools compliance [OPTIONS] -j <inspec-json> -f <threshold-file>
 FLAGS:
 	-j --inspec-json <inspec-json>          : path to InSpec results Json
 	-i --template-inline <threshold-inline> : inline compliance threshold definition
-	-f --template-file <threshold-file>     : yaml file with compliance threshold definition
+	-f --threshold-file <threshold-file>    : yaml file with compliance threshold definition
 Examples:
 
   inspec_tools compliance -j examples/sample_json/rhel-simp.json -i '{compliance.min: 80, failed.critical.max: 0, failed.high.max: 0}'
@@ -135,11 +137,11 @@ failed.high.max: 1
 
 #### In-Line Examples
 ```
-{compliance: {min: 90}, failed: {critical: {max: 0}, high: {max: 0}}}
+"{compliance: {min: 90}, failed: {critical: {max: 0}, high: {max: 0}}}"
 ```
 
 ```
-{compliance.min: 81, failed.critical.max: 0, failed.high.max: 0}
+"{compliance.min: 81, failed.critical.max: 0, failed.high.max: 0}"
 ```
 
 ## summary
@@ -185,12 +187,16 @@ Using additional flags will override the normal output and only display the outp
 
 USAGE: inspec_tools summary [OPTIONS] -j <inspec-json>
 
+Thresholds provided inline (i.e. `-i`) override thresholds provided by files (i.e. `-t`).
+
+
 ```
 FLAGS:
-  -j --inspec-json <inspec-json>   : path to InSpec results JSON
-  -V --verbose, --no-verbose       : print verbose an debug output
-  -f --json-full, --no-json-full   : print the summary STDOUT as JSON
-  -k --json-counts, --no-json_cou  : print the reslut status to STDOUT as JSON
+  -j --inspec-json <inspec-json>           : path to InSpec results JSON
+  -f --json-full, --no-json-full           : print the summary STDOUT as JSON
+  -k --json-counts, --no-json-counts       : print the result status to STDOUT as JSON
+  -t, --threshold-file=THRESHOLD_FILE]     : path to threshold YAML file
+  -i, --threshold-inline=THRESHOLD_INLINE] : string of text representing threshold YAML inline
 
 Examples:
 
@@ -220,16 +226,18 @@ example: inspec_tools xccdf2inspec -x xccdf_file.xml -a attributes.yml -o myprof
 
 `inspec2xccdf` converts an InSpec profile in json format to a STIG XCCDF Document
 
+See [examples documentation](./examples/inspec2xccdf/README.md) for additional guidance on usage including attribute details.
+
 ```
 USAGE: inspec_tools inspec2xccdf [OPTIONS] -j <inspec-json> -a <xccdf-attr-yml> -o <xccdf-xml>
 
 FLAGS:
 	-j --inspec-json <inspec-json>   : path to InSpec Json file created using command 'inspec json <profile> > example.json'
-	-a --attributes <xccdf-attr-yml> : path to yml file that provides the required attributes for the XCCDF Document. these attributes are parts of XCCDF document which do not fit into the InSpec schema
-	-o --output <xccdf-xml>          : name or path to create the xccdf and title to give the xccdf
-	-V --verbose                     : verbose run [optional]
+	-a --attributes <xccdf-attr-yml> : path to yml file that provides the required attributes for the XCCDF document. These attributes are parts of XCCDF document which do not fit into the InSpec schema.
+	-o --output <xccdf-xml>          : name or path to create the XCCDF and title to give the XCCDF
+        -m, [--metadata=METADATA]        : path to json file with additional host metadata for the XCCDF file
 
-example: inspec_tools inspec2xccdf -j example.json -a attributes.yml -o xccdf.xml
+example: inspec_tools inspec2xccdf -j examples/sample_json/good_nginxresults.json -a lib/data/attributes.yml -o output.xccdf
 ```
 
 ## csv2inspec
@@ -295,7 +303,7 @@ FLAGS:
 	-s --separate-files [true | false] : output the resulting controls as multiple files (default: true) [optional]
 	-d --debug                         : debug run [optional]
 
-example: inspec_tools pdf2inspec -p benchmark.pdf -o /path/to/myprofile -f ruby -s true
+example: inspec_tools pdf2inspec -p examples/CIS_Ubuntu_Linux_16.04_LTS_Benchmark_v1.0.0.pdf -o /path/to/myprofile -f ruby -s true
 ```
 
 ## xlsx2inspec

data/Rakefile

@@ -1,10 +1,9 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
+require 'rake/testtask'
 require File.expand_path('../lib/inspec_tools/version', __FILE__)
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
+  t.libs << 'test'
+  t.libs << 'lib'
   t.test_files = FileList['test/**/*_test.rb']
 end
 
@@ -20,11 +19,86 @@ namespace :test do
       'test/unit/inspec_tools_test.rb'
     ])
   end
+
+  Rake::TestTask.new(:exclude_slow) do |t|
+    t.description = 'Excluding all tests that take more than 3 seconds to complete'
+    t.libs << 'test'
+    t.libs << "lib"
+    t.verbose = true
+    t.test_files = FileList['test/**/*_test.rb'].reject{|file| file.include? 'pdf_test.rb'}.reverse
+  end
+end 
+
+desc 'Build for release'
+task :build_release do
+
+  Rake::Task["generate_mapping_objects"].reenable
+  Rake::Task["generate_mapping_objects"].invoke
+
+  system('gem build inspec_tools.gemspec')
+end
+
+desc 'Generate mapping objects'
+task :generate_mapping_objects do
+  require 'roo'
+
+  nist_mapping_cis_controls = ENV['NIST_MAPPING_CIS_CONTROLS'] || 'NIST_Map_02052020_CIS_Controls_Version_7.1_Implementation_Groups_1.2.xlsx'.freeze
+  nist_mapping_cis_critical_controls = ENV['NIST_MAPPING_CIS_CRITICAL_CONTROLS'] || 'NIST_Map_09212017B_CSC-CIS_Critical_Security_Controls_VER_6.1_Excel_9.1.2016.xlsx'.freeze
+
+  data_root_path = File.join(File.expand_path(__dir__), 'lib', 'data')
+  cis_controls_path = File.join(data_root_path, nist_mapping_cis_controls)
+  cis_critical_controls_path = File.join(data_root_path, nist_mapping_cis_critical_controls)
+
+  raise "#{cis_controls_path} does not exist" unless File.exist?(cis_controls_path)
+
+  raise "#{cis_critical_controls_path} does not exist" unless File.exist?(cis_critical_controls_path)
+
+  marshal_cis_controls(cis_controls_path, data_root_path)
+  marshal_cis_critical_controls(cis_critical_controls_path, data_root_path)
 end
 
-desc 'Build and publish the gem'
-task publish: :build do
-  system("gem push pkg/inspec_tools-#{InspecTools::VERSION}.gem")
+def marshal_cis_controls(cis_controls_path, data_root_path)
+  cis_to_nist = {}
+  Roo::Spreadsheet.open(cis_controls_path).sheet(3).each do |row|
+    if row[3].is_a?(Numeric)
+      cis_to_nist[row[3].to_s] = row[0]
+    else
+      cis_to_nist[row[2].to_s] = row[0] unless (row[2] == '') || row[2].to_i.nil?
+    end
+  end
+  output_file = File.new(File.join(data_root_path, 'cis_to_nist_mapping'), 'w')
+  Marshal.dump(cis_to_nist, output_file)
+  output_file.close
 end
 
-task :default => :test
+def marshal_cis_critical_controls(cis_critical_controls_path, data_root_path)
+  controls_spreadsheet = Roo::Spreadsheet.open(cis_critical_controls_path)
+  controls_spreadsheet.default_sheet = 'VER 6.1 Controls'
+  headings = {}
+  controls_spreadsheet.row(3).each_with_index { |header, idx| headings[header] = idx }
+
+  nist_ver = 4
+  cis_ver = controls_spreadsheet.row(2)[4].split(' ')[-1]
+  control_count = 1
+  mapping = []
+  ((controls_spreadsheet.first_row + 3)..controls_spreadsheet.last_row).each do |row_value|
+    current_row = {}
+    if controls_spreadsheet.row(row_value)[headings['NIST SP 800-53 Control #']].to_s != ''
+      current_row[:nist] = controls_spreadsheet.row(row_value)[headings['NIST SP 800-53 Control #']].to_s
+    else
+      current_row[:nist] = 'Not Mapped'
+    end
+    current_row[:nist_ver] = nist_ver
+    if controls_spreadsheet.row(row_value)[headings['Control']].to_s == ''
+      current_row[:cis] = control_count.to_s
+      control_count += 1
+    else
+      current_row[:cis] = controls_spreadsheet.row(row_value)[headings['Control']].to_s
+    end
+    current_row[:cis_ver] = cis_ver
+    mapping << current_row
+  end
+  output_file = File.new(File.join(data_root_path, 'cis_to_nist_critical_controls'), 'w')
+  Marshal.dump(mapping, output_file)
+  output_file.close
+end

data/lib/happy_mapper_tools/benchmark.rb

@@ -99,6 +99,88 @@ module HappyMapperTools
       element :content, String, tag: 'check-content'
     end
 
+    class MessageType
+      include HappyMapper
+      attribute :severity, String, tag: 'severity'
+      content :message, String
+    end
+
+    class RuleResultType
+      include HappyMapper
+      attribute :idref, String, tag: 'idref'
+      attribute :severity, String, tag: 'severity'
+      attribute :time, String, tag: 'time'
+      attribute :weight, String, tag: 'weight'
+      element :result, String, tag: 'result'
+      # element override - Not implemented. Does not apply to Inspec execution
+      has_many :ident, Ident, tag: 'ident'
+      # Note: element metadata not implemented at this time
+      has_many :message, MessageType, tag: 'message'
+      has_many :instance, String, tag: 'instance'
+      element :fix, Fix, tag: 'fix'
+      element :check, Check, tag: 'check'
+    end
+
+    class ScoreType
+      include HappyMapper
+
+      def initialize(system, maximum, score)
+        @system = system
+        @maximum = maximum
+        @score = score
+      end
+
+      attribute :system, String, tag: 'system'
+      attribute :maximum, String, tag: 'maximum' # optional attribute
+      content :score, String
+    end
+
+    class CPE2idrefType
+      include HappyMapper
+      attribute :idref, String, tag: 'idref'
+    end
+
+    class IdentityType
+      include HappyMapper
+      attribute :authenticated, Boolean, tag: 'authenticated'
+      attribute :privileged, Boolean, tag: 'privileged'
+      content :identity, String
+    end
+
+    class Fact
+      include HappyMapper
+      attribute :name, String, tag: 'name'
+      attribute :type, String, tag: 'type'
+      content :fact, String
+    end
+
+    class TargetFact
+      include HappyMapper
+      has_many :fact, Fact, tag: 'fact'
+    end
+
+    class TestResult
+      include HappyMapper
+      # Note: element benchmark not implemented at this time since this is same file
+      # Note: element title not implemented due to no mapping from Chef Inspec
+      element :remark, String, tag: 'remark'
+      has_many :organization, String, tag: 'organization'
+      element :identity, IdentityType, tag: 'identity'
+      element :target, String, tag: 'target'
+      has_many :target_address, String, tag: 'target-address'
+      element :target_facts, TargetFact, tag: 'target-facts'
+      element :platform, CPE2idrefType, tag: 'platform'
+      # Note: element profile not implemented since Benchmark profile is also not implemented
+      has_many :rule_result, RuleResultType, tag: 'rule-result'
+      has_many :score, ScoreType, tag: 'score' # One minimum
+      # Note: element signature not implemented due to no mapping from Chef Inspec
+      attribute :id, String, tag: 'id'
+      attribute :starttime, String, tag: 'start-time'
+      attribute :endtime, String, tag: 'end-time'
+      # Note: attribute test-system not implemented at this time due to unknown CPE value for Chef Inspec
+      attribute :version, String, tag: 'version'
+    end
+
     # Class Profile maps from the 'Profile' from Benchmark XML file using HappyMapper
     class Profile
       include HappyMapper
@@ -156,6 +238,7 @@ module HappyMapperTools
       element :version, String, tag: 'version'
       has_many :profile, Profile, tag: 'Profile'
       has_many :group, Group, tag: 'Group'
+      element :testresult, TestResult, tag: 'TestResult'
     end
   end
 end

data/lib/inspec_tools/csv.rb

@@ -1,10 +1,10 @@
 require 'csv'
-require 'nokogiri'
-require 'word_wrap'
 require 'yaml'
 require 'digest'
 
 require_relative '../utilities/inspec_util'
+require_relative '../utilities/cci_xml'
+require_relative '../utilities/mapping_validator'
 
 # rubocop:disable Metrics/AbcSize
 # rubocop:disable Metrics/PerceivedComplexity
@@ -16,34 +16,25 @@ module InspecTools
     def initialize(csv, mapping, name, verbose = false)
       @name = name
       @csv = csv
-      @mapping = mapping
+      @mapping = Utils::MappingValidator.validate(mapping)
       @verbose = verbose
       @csv.shift if @mapping['skip_csv_header']
     end
 
-    def to_ckl
-      # TODO
-    end
-
-    def to_xccdf
-      # TODO
-    end
-
-    def to_inspec
+    def to_inspec(control_name_prefix: nil)
       @controls = []
-      @cci_xml = nil
       @profile = {}
-      read_cci_xml
-      insert_json_metadata
-      parse_controls
+      @cci_xml = Utils::CciXml.get_cci_list('U_CCI_List.xml')
+      insert_metadata
+      parse_controls(control_name_prefix)
       @profile['controls'] = @controls
-      @profile['sha256'] = Digest::SHA256.hexdigest @profile.to_s
+      @profile['sha256'] = Digest::SHA256.hexdigest(@profile.to_s)
       @profile
     end
 
     private
 
-    def insert_json_metadata
+    def insert_metadata
       @profile['name'] = @name
       @profile['title'] = 'InSpec Profile'
       @profile['maintainer'] = 'The Authors'
@@ -60,35 +51,37 @@ module InspecTools
       }
     end
 
-    def read_cci_xml
-      cci_list_path = File.join(File.dirname(__FILE__), '../data/U_CCI_List.xml')
-      @cci_xml = Nokogiri::XML(File.open(cci_list_path))
-      @cci_xml.remove_namespaces!
-    rescue StandardError => e
-      puts "Exception: #{e.message}"
-    end
-
     def get_nist_reference(cci_number)
       item_node = @cci_xml.xpath("//cci_list/cci_items/cci_item[@id='#{cci_number}']")[0] unless @cci_xml.nil?
-      unless item_node.nil?
-        nist_ref = item_node.xpath('./references/reference[not(@version <= preceding-sibling::reference/@version) and not(@version <=following-sibling::reference/@version)]/@index').text
-        nist_ver = item_node.xpath('./references/reference[not(@version <= preceding-sibling::reference/@version) and not(@version <=following-sibling::reference/@version)]/@version').text
-      end
-      [nist_ref, nist_ver]
+      return nil if item_node.nil?
+
+      [] << item_node.xpath('./references/reference[not(@version <= preceding-sibling::reference/@version) and not(@version <=following-sibling::reference/@version)]/@index').text
     end
 
-    def parse_controls
+    def get_cci_number(cell)
+      # Return nil if a mapping to the CCI was not provided or if there is not content in the CSV cell.
+      return nil if cell.nil? || @mapping['control.tags']['cci'].nil?
+
+      # If the content has been exported from STIG Viewer, the cell will have extra information
+      cell.split("\n").first
+    end
+
+    def parse_controls(prefix)
       @csv.each do |row|
-        print '.'
         control = {}
-        control['id']     = row[@mapping['control.id']]     unless @mapping['control.id'].nil? || row[@mapping['control.id']].nil?
-        control['title']  = row[@mapping['control.title']]  unless @mapping['control.title'].nil? || row[@mapping['control.title']].nil?
-        control['desc']   = row[@mapping['control.desc']]   unless @mapping['control.desc'].nil? || row[@mapping['control.desc']].nil?
+        control['id']     = generate_control_id(prefix, row[@mapping['control.id']]) unless @mapping['control.id'].nil? || row[@mapping['control.id']].nil?
+        control['title']  = row[@mapping['control.title']] unless @mapping['control.title'].nil? || row[@mapping['control.title']].nil?
+        control['desc']   = row[@mapping['control.desc']] unless @mapping['control.desc'].nil? || row[@mapping['control.desc']].nil?
         control['tags'] = {}
-        nist, nist_rev = get_nist_reference(row[@mapping['control.tags']['cci']]) unless @mapping['control.tags']['cci'].nil? || row[@mapping['control.tags']['cci']].nil?
-        control['tags']['nist'] = [nist, 'Rev_' + nist_rev] unless nist.nil? || nist_rev.nil?
+        cci_number = get_cci_number(row[@mapping['control.tags']['cci']])
+        nist = get_nist_reference(cci_number) unless cci_number.nil?
+        control['tags']['nist'] = nist unless nist.nil? || nist.include?(nil)
         @mapping['control.tags'].each do |tag|
-          control['tags'][tag.first.to_s] = row[tag.last] unless row[tag.last].nil?
+          if tag.first == 'cci'
+            control['tags'][tag.first] = cci_number
+            next
+          end
+          control['tags'][tag.first] = row[tag.last] unless row[tag.last].nil?
         end
         unless @mapping['control.tags']['severity'].nil? || row[@mapping['control.tags']['severity']].nil?
           control['impact'] = Utils::InspecUtil.get_impact(row[@mapping['control.tags']['severity']])
@@ -97,5 +90,15 @@ module InspecTools
         @controls << control
       end
     end
+
+    def generate_control_id(prefix, id)
+      return id if prefix.nil?
+
+      "#{prefix}-#{id}"
+    end
   end
 end
+
+# rubocop:enable Metrics/AbcSize
+# rubocop:enable Metrics/PerceivedComplexity
+# rubocop:enable Metrics/CyclomaticComplexity

data/lib/inspec_tools/generate_map.rb

@@ -0,0 +1,35 @@
+module InspecTools
+  class GenerateMap
+    attr_accessor :text
+
+    def initialize(text = nil)
+      @text = text.nil? ? default_text : text
+    end
+
+    def generate_example(file)
+      File.write(file, @text)
+    end
+
+    private
+
+    def default_text
+      <<~YML
+      # Setting csv_header to true will skip the csv file header
+      skip_csv_header: true
+      width   : 80
+
+
+      control.id: 0
+      control.title: 15
+      control.desc: 16
+      control.tags:
+        severity: 1
+        rid: 8
+        stig_id: 3
+        cci: 2
+        check: 12
+        fix: 10
+      YML
+    end
+  end
+end