heimdall_tools 1.3.37 → 1.3.41

data/lib/heimdall_tools/help/netsparker_mapper.md

@@ -0,0 +1,7 @@
+  netsparker_mapper translates an Netsparker  XML results file into HDF format json to be viewable in Heimdall
+
+  The current iteration only works with Netsparker Enterprise Vulnerabilities Scan.
+  
+Examples:
+
+  heimdall_tools netsparker_mapper -x netsparker_results.xml -o netsparker_hdf.json

data/lib/heimdall_tools/jfrog_xray_mapper.rb

@@ -10,10 +10,10 @@ CWE_NIST_MAPPING_FILE = File.join(RESOURCE_DIR, 'cwe-nist-mapping.csv')
 IMPACT_MAPPING = {
   high: 0.7,
   medium: 0.5,
-  low: 0.3,
+  low: 0.3
 }.freeze
 
-DEFAULT_NIST_TAG = ["SA-11", "RA-5"].freeze
+DEFAULT_NIST_TAG = %w{SA-11 RA-5}.freeze
 
 # Loading spinner sign
 $spinner = Enumerator.new do |e|
@@ -27,14 +27,13 @@ end
 
 module HeimdallTools
   class JfrogXrayMapper
-    def initialize(xray_json, name=nil, verbose = false)
+    def initialize(xray_json, _name = nil, verbose = false)
       @xray_json = xray_json
       @verbose = verbose
 
       begin
         @cwe_nist_mapping = parse_mapper
         @project = JSON.parse(xray_json)
-
       rescue StandardError => e
         raise "Invalid JFrog Xray JSON file provided Exception: #{e}"
       end
@@ -44,11 +43,11 @@ module HeimdallTools
       finding = {}
       finding['status'] = 'failed'
       finding['code_desc'] = []
-      finding['code_desc'] << "source_comp_id : #{vulnerability['source_comp_id'].to_s }"
-      finding['code_desc'] << "vulnerable_versions : #{vulnerability['component_versions']['vulnerable_versions'].to_s }"
-      finding['code_desc'] << "fixed_versions : #{vulnerability['component_versions']['fixed_versions'].to_s }"
-      finding['code_desc'] << "issue_type : #{vulnerability['issue_type'].to_s }"
-      finding['code_desc'] << "provider : #{vulnerability['provider'].to_s }"
+      finding['code_desc'] << "source_comp_id : #{vulnerability['source_comp_id']}"
+      finding['code_desc'] << "vulnerable_versions : #{vulnerability['component_versions']['vulnerable_versions']}"
+      finding['code_desc'] << "fixed_versions : #{vulnerability['component_versions']['fixed_versions']}"
+      finding['code_desc'] << "issue_type : #{vulnerability['issue_type']}"
+      finding['code_desc'] << "provider : #{vulnerability['provider']}"
       finding['code_desc'] = finding['code_desc'].join("\n")
       finding['run_time'] = NA_FLOAT
 
@@ -57,17 +56,25 @@ module HeimdallTools
       [finding]
     end
 
+    def format_control_desc(vulnerability)
+      text = []
+      info = vulnerability['component_versions']['more_details']
+      text << info['description'].to_s
+      text << "cves: #{info['cves']}" unless info['cves'].nil?
+      text.join('<br>')
+    end
+
     def nist_tag(cweid)
-      entries = @cwe_nist_mapping.select { |x| cweid.include? x[:cweid].to_s }
+      entries = @cwe_nist_mapping.select { |x| cweid.include?(x[:cweid].to_s) && !x[:nistid].nil? }
       tags = entries.map { |x| x[:nistid] }
       tags.empty? ? DEFAULT_NIST_TAG : tags.flatten.uniq
     end
 
     def parse_identifiers(vulnerability, ref)
       # Extracting id number from reference style CWE-297
-      vulnerability['component_versions']['more_details']['cves'][0][ref.downcase].map { |e| e.split("#{ref}-")[1]  }
-      rescue
-        return []
+      vulnerability['component_versions']['more_details']['cves'][0][ref.downcase].map { |e| e.split("#{ref}-")[1] }
+    rescue StandardError
+      []
     end
 
     def impact(severity)
@@ -83,17 +90,17 @@ module HeimdallTools
     end
 
     def desc_tags(data, label)
-      { "data": data || NA_STRING, "label": label || NA_STRING }
+      { data: data || NA_STRING, label: label || NA_STRING }
     end
 
     # Xray report could have multiple vulnerability entries for multiple findings of same issue type.
-    # The meta data is identical across entries 
+    # The meta data is identical across entries
     # method collapse_duplicates return unique controls with applicable findings collapsed into it.
     def collapse_duplicates(controls)
       unique_controls = []
 
       controls.map { |x| x['id'] }.uniq.each do |id|
-        collapsed_results = controls.select { |x| x['id'].eql?(id) }.map {|x| x['results']}
+        collapsed_results = controls.select { |x| x['id'].eql?(id) }.map { |x| x['results'] }
         unique_control = controls.find { |x| x['id'].eql?(id) }
         unique_control['results'] = collapsed_results.flatten
         unique_controls << unique_control
@@ -104,9 +111,9 @@ module HeimdallTools
     def to_hdf
       controls = []
       vulnerability_count = 0
-      @project['data'].uniq.each do | vulnerability |
+      @project['data'].uniq.each do |vulnerability|
         printf("\rProcessing: %s", $spinner.next)
-        
+
         vulnerability_count +=1
         item = {}
         item['tags']               = {}
@@ -115,26 +122,26 @@ module HeimdallTools
         item['source_location']    = NA_HASH
         item['descriptions']       = NA_ARRAY
 
-        # Xray JSONs might note have `id` fields populated. 
+        # Xray JSONs might note have `id` fields populated.
         # If thats a case MD5 hash is used to collapse vulnerability findings of the same type.
-        item['id']                 = vulnerability['id'].empty? ? OpenSSL::Digest::MD5.digest(vulnerability['summary'].to_s).unpack("H*")[0].to_s : vulnerability['id']
+        item['id']                 = vulnerability['id'].empty? ? OpenSSL::Digest::MD5.digest(vulnerability['summary'].to_s).unpack1('H*').to_s : vulnerability['id']
         item['title']              = vulnerability['summary'].to_s
-        item['desc']               = vulnerability['component_versions']['more_details']['description'].to_s
-        item['impact']             = impact(vulnerability['severity'].to_s) 
+        item['desc']               = format_control_desc(vulnerability)
+        item['impact']             = impact(vulnerability['severity'].to_s)
         item['code']               = NA_STRING
         item['results']            = finding(vulnerability)
 
-        item['tags']['nist']       = nist_tag( parse_identifiers( vulnerability, 'CWE') )
-        item['tags']['cweid']      = parse_identifiers( vulnerability, 'CWE')
+        item['tags']['nist']       = nist_tag(parse_identifiers(vulnerability, 'CWE'))
+        item['tags']['cweid']      = parse_identifiers(vulnerability, 'CWE')
 
         controls << item
       end
 
       controls = collapse_duplicates(controls)
-      results = HeimdallDataFormat.new(profile_name: "JFrog Xray Scan",
+      results = HeimdallDataFormat.new(profile_name: 'JFrog Xray Scan',
                                        version: NA_STRING,
-                                       title: "JFrog Xray Scan", 
-                                       summary: "Continuous Security and Universal Artifact Analysis",
+                                       title: 'JFrog Xray Scan',
+                                       summary: 'Continuous Security and Universal Artifact Analysis',
                                        controls: controls)
       results.to_hdf
     end

data/lib/heimdall_tools/nessus_mapper.rb

@@ -6,7 +6,7 @@ require 'nokogiri'
 
 RESOURCE_DIR = Pathname.new(__FILE__).join('../../data')
 
-NESSUS_PLUGINS_NIST_MAPPING_FILE =   File.join(RESOURCE_DIR, 'nessus-plugins-nist-mapping.csv')
+NESSUS_PLUGINS_NIST_MAPPING_FILE = File.join(RESOURCE_DIR, 'nessus-plugins-nist-mapping.csv')
 U_CCI_LIST =   File.join(RESOURCE_DIR, 'U_CCI_List.xml')
 
 IMPACT_MAPPING = {
@@ -14,16 +14,16 @@ IMPACT_MAPPING = {
   Low: 0.3,
   Medium: 0.5,
   High: 0.7,
-  Critical: 0.9,
+  Critical: 0.9
 }.freeze
 
-DEFAULT_NIST_TAG = ["unmapped"].freeze
+DEFAULT_NIST_TAG = ['unmapped'].freeze
 
 # Nessus results file 800-53 refs does not contain Nist rev version. Using this default
 # version in that case
 DEFAULT_NIST_REV = 'Rev_4'.freeze
 
-NA_PLUGIN_OUTPUT = "This Nessus Plugin does not provide output message.".freeze
+NA_PLUGIN_OUTPUT = 'This Nessus Plugin does not provide output message.'.freeze
 
 # rubocop:disable Metrics/AbcSize
 
@@ -51,19 +51,16 @@ module HeimdallTools
       rescue StandardError => e
         raise "Invalid Nessus XML file provided Exception: #{e}"
       end
-
     end
 
     def extract_report
-      begin
-        # When there are multiple hosts in the nessus report ReportHost field is an array
-        # When there is only one host in the nessus report ReportHost field is a hash
-        # Array() converts ReportHost to array in case there is only one host
-        reports = @data['NessusClientData_v2']['Report']['ReportHost']
-        reports.kind_of?(Array) ? reports : [reports]
-      rescue StandardError => e
-        raise "Invalid Nessus XML file provided Exception: #{e}"
-      end
+      # When there are multiple hosts in the nessus report ReportHost field is an array
+      # When there is only one host in the nessus report ReportHost field is a hash
+      # Array() converts ReportHost to array in case there is only one host
+      reports = @data['NessusClientData_v2']['Report']['ReportHost']
+      reports.is_a?(Array) ? reports : [reports]
+    rescue StandardError => e
+      raise "Invalid Nessus XML file provided Exception: #{e}"
     end
 
     def parse_refs(refs, key)
@@ -71,24 +68,20 @@ module HeimdallTools
     end
 
     def extract_scaninfo
-      begin
-        policy = @data['NessusClientData_v2']['Policy']
-        info = {}
+      policy = @data['NessusClientData_v2']['Policy']
+      info = {}
 
-        info['policyName'] = policy['policyName']
-        info['version'] = policy['Preferences']['ServerPreferences']['preference'].select {|x| x['name'].eql? 'sc_version'}.first['value']
-        info
-      rescue StandardError => e
-        raise "Invalid Nessus XML file provided Exception: #{e}"
-      end
+      info['policyName'] = policy['policyName']
+      info['version'] = policy['Preferences']['ServerPreferences']['preference'].select { |x| x['name'].eql? 'sc_version' }.first['value']
+      info
+    rescue StandardError => e
+      raise "Invalid Nessus XML file provided Exception: #{e}"
     end
 
     def extract_timestamp(report)
-      begin
-        timestamp = report['HostProperties']['tag'].select {|x| x['name'].eql? 'HOST_START'}.first['text']
-      rescue StandardError => e
-        raise "Invalid Nessus XML file provided Exception: #{e}"
-      end
+      report['HostProperties']['tag'].select { |x| x['name'].eql? 'HOST_START' }.first['text']
+    rescue StandardError => e
+      raise "Invalid Nessus XML file provided Exception: #{e}"
     end
 
     def format_desc(issue)
@@ -129,7 +122,7 @@ module HeimdallTools
 
     def cci_nist_tag(cci_refs)
       nist_tags = []
-      cci_refs.each do | cci_ref |
+      cci_refs.each do |cci_ref|
         item_node = @cci_xml.xpath("//cci_list/cci_items/cci_item[@id='#{cci_ref}']")[0] unless @cci_xml.nil?
         unless item_node.nil?
           nist_ref = item_node.xpath('./references/reference[not(@version <= preceding-sibling::reference/@version) and not(@version <=following-sibling::reference/@version)]/@index').text
@@ -140,7 +133,7 @@ module HeimdallTools
     end
 
     def plugin_nist_tag(pluginfamily, pluginid)
-      entries = @cwe_nist_mapping.select { |x| (x[:pluginfamily].eql?(pluginfamily) && (x[:pluginid].eql?('*') || x[:pluginid].eql?(pluginid.to_i)) ) }
+      entries = @cwe_nist_mapping.select { |x| (x[:pluginfamily].eql?(pluginfamily) && (x[:pluginid].eql?('*') || x[:pluginid].eql?(pluginid.to_i))) && !x[:nistid].nil? }
       tags = entries.map { |x| [x[:nistid].split('|'), "Rev_#{x[:rev]}"] }
       tags.empty? ? DEFAULT_NIST_TAG : tags.flatten.uniq
     end
@@ -148,15 +141,15 @@ module HeimdallTools
     def impact(severity)
       # Map CAT levels and Plugin severity to HDF impact levels
       case severity
-      when "0"
+      when '0'
         IMPACT_MAPPING[:Info]
-      when "1","III"
+      when '1', 'III'
         IMPACT_MAPPING[:Low]
-      when "2","II"
+      when '2', 'II'
         IMPACT_MAPPING[:Medium]
-      when "3","I"
+      when '3', 'I'
         IMPACT_MAPPING[:High]
-      when "4"
+      when '4'
         IMPACT_MAPPING[:Critical]
       else
         -1
@@ -172,17 +165,17 @@ module HeimdallTools
     end
 
     def desc_tags(data, label)
-      { "data": data || NA_STRING, "label": label || NA_STRING }
+      { data: data || NA_STRING, label: label || NA_STRING }
     end
 
     # Nessus report could have multiple issue entries for multiple findings of same issue type.
-    # The meta data is identical across entries 
+    # The meta data is identical across entries
     # method collapse_duplicates return unique controls with applicable findings collapsed into it.
     def collapse_duplicates(controls)
       unique_controls = []
 
       controls.map { |x| x['id'] }.uniq.each do |id|
-        collapsed_results = controls.select { |x| x['id'].eql?(id) }.map {|x| x['results']}
+        collapsed_results = controls.select { |x| x['id'].eql?(id) }.map { |x| x['results'] }
         unique_control = controls.find { |x| x['id'].eql?(id) }
         unique_control['results'] = collapsed_results.flatten
         unique_controls << unique_control
@@ -192,9 +185,9 @@ module HeimdallTools
 
     def to_hdf
       host_results = {}
-      @reports.each do | report|
+      @reports.each do |report|
         controls = []
-        report['ReportItem'].each do | item |
+        report['ReportItem'].each do |item|
           printf("\rProcessing: %s", $spinner.next)
           @item = {}
           @item['tags']               = {}
@@ -207,7 +200,7 @@ module HeimdallTools
           # Current version covers STIG based 'Policy Compliance' results
           # TODO Cover cases for 'Policy Compliance' results based on CIS
           if item['compliance-reference']
-            @item['id']                 = parse_refs(item['compliance-reference'],'Vuln-ID').join.to_s
+            @item['id']                 = parse_refs(item['compliance-reference'], 'Vuln-ID').join.to_s
           else
             @item['id']                 = item['pluginID'].to_s
           end
@@ -222,17 +215,17 @@ module HeimdallTools
             @item['desc']              = format_desc(item).to_s
           end
           if item['compliance-reference']
-            @item['impact']            = impact(parse_refs(item['compliance-reference'],'CAT').join.to_s)
+            @item['impact']            = impact(parse_refs(item['compliance-reference'], 'CAT').join.to_s)
           else
-            @item['impact']            = impact(item['severity']) 
+            @item['impact']            = impact(item['severity'])
           end
           if item['compliance-reference']
-            @item['tags']['nist']     = cci_nist_tag(parse_refs(item['compliance-reference'],'CCI'))
+            @item['tags']['nist']     = cci_nist_tag(parse_refs(item['compliance-reference'], 'CCI'))
           else
-            @item['tags']['nist']     = plugin_nist_tag(item['pluginFamily'],item['pluginID'])
+            @item['tags']['nist']     = plugin_nist_tag(item['pluginFamily'], item['pluginID'])
           end
           if item['compliance-solution']
-            @item['descriptions']       <<  desc_tags(item['compliance-solution'], 'check')
+            @item['descriptions']       << desc_tags(item['compliance-solution'], 'check')
           end
 
           @item['code']               = ''

data/lib/heimdall_tools/netsparker_mapper.rb

@@ -0,0 +1,164 @@
+require 'json'
+require 'csv'
+require 'heimdall_tools/hdf'
+require 'utilities/xml_to_hash'
+
+RESOURCE_DIR = Pathname.new(__FILE__).join('../../data')
+
+CWE_NIST_MAPPING_FILE = File.join(RESOURCE_DIR, 'cwe-nist-mapping.csv')
+OWASP_NIST_MAPPING_FILE = File.join(RESOURCE_DIR, 'owasp-nist-mapping.csv')
+
+IMPACT_MAPPING = {
+  Critical: 1.0,
+  High: 0.7,
+  Medium: 0.5,
+  Low: 0.3,
+  Best_Practice: 0.0,
+  Information: 0.0
+}.freeze
+
+DEFAULT_NIST_TAG = %w{SA-11 RA-5}.freeze
+
+module HeimdallTools
+  class NetsparkerMapper
+    def initialize(xml, _name = nil, verbose = false)
+      @verbose = verbose
+
+      begin
+        @cwe_nist_mapping = parse_mapper(CWE_NIST_MAPPING_FILE)
+        @owasp_nist_mapping = parse_mapper(OWASP_NIST_MAPPING_FILE)
+        data = xml_to_hash(xml)
+
+        @vulnerabilities = data['netsparker-enterprise']['vulnerabilities']['vulnerability']
+        @scan_info = data['netsparker-enterprise']['target']
+      rescue StandardError => e
+        raise "Invalid Netsparker XML file provided Exception: #{e}"
+      end
+    end
+
+    def to_hdf
+      controls = []
+      @vulnerabilities.each do |vulnerability|
+        @item = {}
+        @item['id']                 = vulnerability['LookupId'].to_s
+        @item['title']              = vulnerability['name'].to_s
+        @item['desc']               = format_control_desc(vulnerability)
+        @item['impact']             = impact(vulnerability['severity'])
+        @item['tags']               = {}
+        @item['descriptions']       = []
+
+        @item['descriptions']       <<  desc_tags(format_check_text(vulnerability), 'check')
+        @item['descriptions']       <<  desc_tags(format_fix_text(vulnerability), 'fix')
+        @item['refs']               = NA_ARRAY
+        @item['source_location']    = NA_HASH
+        @item['tags']['nist']       = nist_tag(vulnerability['classification'])
+        @item['code']               = ''
+        @item['results']            = finding(vulnerability)
+
+        controls << @item
+      end
+      controls = collapse_duplicates(controls)
+      results = HeimdallDataFormat.new(profile_name: 'Netsparker Enterprise Scan',
+                                       title: "Netsparker Enterprise Scan ID: #{@scan_info['scan-id']} URL: #{@scan_info['url']}",
+                                       summary: 'Netsparker Enterprise Scan',
+                                       target_id: @scan_info['url'],
+                                       controls: controls)
+      results.to_hdf
+    end
+
+    private
+
+    def parse_html(block)
+      block['#cdata-section'].to_s.strip unless block.nil?
+    end
+
+    def finding(vulnerability)
+      finding = {}
+      finding['status'] = 'failed'
+      finding['code_desc'] = []
+      finding['code_desc'] << "http-request : #{parse_html(vulnerability['http-request']['content'])}"
+      finding['code_desc'] << "method : #{vulnerability['http-request']['method']}"
+      finding['code_desc'] = finding['code_desc'].join("\n")
+
+      finding['message'] = []
+      finding['message'] << "http-response : #{parse_html(vulnerability['http-response']['content'])}"
+      finding['message'] << "duration : #{vulnerability['http-response']['duration']}"
+      finding['message'] << "status-code : #{vulnerability['http-response']['status-code']}"
+      finding['message'] = finding['message'].join("\n")
+      finding['run_time'] = NA_FLOAT
+
+      finding['start_time'] = @scan_info['initiated']
+      [finding]
+    end
+
+    def format_control_desc(vulnerability)
+      text = []
+      text << parse_html(vulnerability['description']).to_s unless vulnerability['description'].nil?
+      text << "Exploitation-skills: #{parse_html(vulnerability['exploitation-skills'])}" unless vulnerability['exploitation-skills'].nil?
+      text << "Extra-information: #{vulnerability['extra-information']}" unless vulnerability['extra-information'].nil?
+      text << "Classification: #{vulnerability['classification']}" unless vulnerability['classification'].nil?
+      text << "Impact: #{parse_html(vulnerability['impact'])}" unless vulnerability['impact'].nil?
+      text << "FirstSeenDate: #{vulnerability['FirstSeenDate']}" unless vulnerability['FirstSeenDate'].nil?
+      text << "LastSeenDate: #{vulnerability['LastSeenDate']}" unless vulnerability['LastSeenDate'].nil?
+      text << "Certainty: #{vulnerability['certainty']}" unless vulnerability['certainty'].nil?
+      text << "Type: #{vulnerability['type']}" unless vulnerability['type'].nil?
+      text << "Confirmed: #{vulnerability['confirmed']}" unless vulnerability['confirmed'].nil?
+      text.join('<br>')
+    end
+
+    def format_check_text(vulnerability)
+      text = []
+      text << "Exploitation-skills: #{parse_html(vulnerability['exploitation-skills'])}" unless vulnerability['exploitation-skills'].nil?
+      text << "Proof-of-concept: #{parse_html(vulnerability['proof-of-concept'])}" unless vulnerability['proof-of-concept'].nil?
+      text.join('<br>')
+    end
+
+    def format_fix_text(vulnerability)
+      text = []
+      text << "Remedial-actions: #{parse_html(vulnerability['remedial-actions'])}" unless vulnerability['remedial-actions'].nil?
+      text << "Remedial-procedure: #{parse_html(vulnerability['remedial-procedure'])}" unless vulnerability['remedial-procedure'].nil?
+      text << "Remedy-references: #{parse_html(vulnerability['remedy-references'])}" unless vulnerability['remedy-references'].nil?
+      text.join('<br>')
+    end
+
+    def nist_tag(classification)
+      tags = []
+      entries = @cwe_nist_mapping.select { |x| classification['cwe'].include?(x[:cweid].to_s) && !x[:nistid].nil? }
+      tags << entries.map { |x| x[:nistid] }
+      entries = @owasp_nist_mapping.select { |x| classification['owasp'].include?(x[:owaspid].to_s) && !x[:nistid].nil? }
+      tags << entries.map { |x| x[:nistid] }
+      tags.flatten.empty? ? DEFAULT_NIST_TAG : tags.flatten.uniq
+    end
+
+    def impact(severity)
+      IMPACT_MAPPING[severity.to_sym]
+    end
+
+    def parse_mapper(mapping_file)
+      csv_data = CSV.read(mapping_file, { encoding: 'UTF-8',
+                                                   headers: true,
+                                                   header_converters: :symbol,
+                                                   converters: :all })
+      csv_data.map(&:to_hash)
+    end
+
+    def desc_tags(data, label)
+      { data: data || NA_STRING, label: label || NA_STRING }
+    end
+
+    # Netsparker report could have multiple issue entries for multiple findings of same issue type.
+    # The meta data is identical across entries
+    # method collapse_duplicates return unique controls with applicable findings collapsed into it.
+    def collapse_duplicates(controls)
+      unique_controls = []
+
+      controls.map { |x| x['id'] }.uniq.each do |id|
+        collapsed_results = controls.select { |x| x['id'].eql?(id) }.map { |x| x['results'] }
+        unique_control = controls.find { |x| x['id'].eql?(id) }
+        unique_control['results'] = collapsed_results.flatten
+        unique_controls << unique_control
+      end
+      unique_controls
+    end
+  end
+end