heimdall_tools 1.3.31 → 1.3.36

data/lib/heimdall_tools.rb

@@ -10,4 +10,8 @@ module HeimdallTools
   autoload :SonarQubeMapper, 'heimdall_tools/sonarqube_mapper'
   autoload :BurpSuiteMapper, 'heimdall_tools/burpsuite_mapper'
   autoload :NessusMapper, 'heimdall_tools/nessus_mapper'
+  autoload :SnykMapper, 'heimdall_tools/snyk_mapper'
+  autoload :NiktoMapper, 'heimdall_tools/nikto_mapper'
+  autoload :JfrogXrayMapper, 'heimdall_tools/jfrog_xray_mapper'
+  autoload :DBProtectMapper, 'heimdall_tools/dbprotect_mapper'
 end

data/lib/heimdall_tools/cli.rb

@@ -61,6 +61,56 @@ module HeimdallTools
       
     end
 
+    desc 'snyk_mapper', 'snyk_mapper translates Snyk results Json to HDF format Json be viewed on Heimdall'
+    long_desc Help.text(:snyk_mapper)
+    option :json, required: true, aliases: '-j'
+    option :output_prefix, required: true, aliases: '-o'
+    option :verbose, type: :boolean, aliases: '-V'
+    def snyk_mapper
+      hdfs = HeimdallTools::SnykMapper.new(File.read(options[:json]), options[:name]).to_hdf
+      puts "\r\HDF Generated:\n"
+      hdfs.keys.each do | host |
+        File.write("#{options[:output_prefix]}-#{host}.json", hdfs[host])
+        puts "#{options[:output_prefix]}-#{host}.json"
+      end
+    end
+
+    desc 'nikto_mapper', 'nikto_mapper translates Nikto results Json to HDF format Json be viewed on Heimdall'
+    long_desc Help.text(:nikto_mapper)
+    option :json, required: true, aliases: '-j'
+    option :output, required: true, aliases: '-o'
+    option :verbose, type: :boolean, aliases: '-V'
+    def nikto_mapper
+      hdf = HeimdallTools::NiktoMapper.new(File.read(options[:json])).to_hdf
+      File.write(options[:output], hdf)
+      puts "\r\HDF Generated:\n"
+      puts "#{options[:output]}"
+    end
+
+    desc 'jfrog_xray_mapper', 'jfrog_xray_mapper translates Jfrog Xray results Json to HDF format Json be viewed on Heimdall'
+    long_desc Help.text(:jfrog_xray_mapper)
+    option :json, required: true, aliases: '-j'
+    option :output, required: true, aliases: '-o'
+    option :verbose, type: :boolean, aliases: '-V'
+    def jfrog_xray_mapper
+      hdf = HeimdallTools::JfrogXrayMapper.new(File.read(options[:json])).to_hdf
+      File.write(options[:output], hdf)
+      puts "\r\HDF Generated:\n"
+      puts "#{options[:output]}"
+    end
+
+    desc 'dbprotect_mapper', 'dbprotect_mapper translates dbprotect results xml to HDF format Json be viewed on Heimdall'
+    long_desc Help.text(:dbprotect_mapper)
+    option :xml, required: true, aliases: '-x'
+    option :output, required: true, aliases: '-o'
+    option :verbose, type: :boolean, aliases: '-V'
+    def dbprotect_mapper
+      hdf = HeimdallTools::DBProtectMapper.new(File.read(options[:xml])).to_hdf
+      File.write(options[:output], hdf)
+      puts "\r\HDF Generated:\n"
+      puts "#{options[:output]}"
+    end
+
     desc 'version', 'prints version'
     def version
       puts VERSION

data/lib/heimdall_tools/dbprotect_mapper.rb

@@ -0,0 +1,127 @@
+require 'json'
+require 'csv'
+require 'heimdall_tools/hdf'
+require 'utilities/xml_to_hash'
+
+IMPACT_MAPPING = {
+  High: 0.7,
+  Medium: 0.5,
+  Low: 0.3,
+  Informational: 0.0
+}.freeze
+
+# rubocop:disable Metrics/AbcSize
+
+module HeimdallTools
+  class DBProtectMapper
+    def initialize(xml, name=nil, verbose = false)
+      @verbose = verbose
+
+      begin
+        dataset = xml_to_hash(xml)
+        @entries = compile_findings(dataset['dataset'])
+
+      rescue StandardError => e
+        raise "Invalid DBProtect XML file provided Exception: #{e};\nNote that XML must be of kind `Check Results Details`."
+      end
+
+    end
+
+    def to_hdf
+      controls = []
+      @entries.each do |entry|
+        @item = {}
+        @item['id']                 = entry['Check ID']
+        @item['title']              = entry['Check']
+        @item['desc']               = format_desc(entry)
+        @item['impact']             = impact(entry['Risk DV'])
+        @item['tags']               = {}
+        @item['descriptions']       = []
+        @item['refs']               = NA_ARRAY
+        @item['source_location']    = NA_HASH
+        @item['code']               = ''
+        @item['results']            = finding(entry)
+
+        controls << @item
+      end
+      controls = collapse_duplicates(controls)
+      results = HeimdallDataFormat.new(profile_name: @entries.first['Policy'],
+                                       version: "",
+                                       title: @entries.first['Job Name'],
+                                       summary: format_summary(@entries.first),
+                                       controls: controls)
+      results.to_hdf
+    end
+
+    private
+
+    def compile_findings(dataset)
+      keys = dataset['metadata']['item'].map{ |e| e['name']}
+      findings = dataset['data']['row'].map { |e| Hash[keys.zip(e['value'])] }
+      findings
+    end
+
+    def format_desc(entry)
+      text = []
+      text << "Task : #{entry['Task']}"
+      text << "Check Category : #{entry['Check Category']}"
+      text.join("; ")
+    end
+
+    def format_summary(entry)
+      text = []
+      text << "Organization : #{entry['Organization']}"
+      text << "Asset : #{entry['Check Asset']}"
+      text << "Asset Type : #{entry['Asset Type']}"
+      text << "IP Address, Port, Instance : #{entry['Asset Type']}"
+      text << "IP Address, Port, Instance : #{entry['IP Address, Port, Instance']}"
+      text.join("\n")
+    end
+
+    def finding(entry)
+      finding = {}
+
+      finding['code_desc'] = entry['Details']
+      finding['run_time'] = 0.0
+      finding['start_time'] = entry['Date']
+
+      case entry['Result Status']
+      when 'Fact'
+        finding['status'] = 'skipped'
+      when 'Failed'
+        finding['status'] = 'failed'
+        finding['backtrace'] = ["DB Protect Failed Check"]
+      when 'Finding'
+        finding['status'] = 'failed'
+      when 'Not A Finding'
+        finding['status'] = 'passed'
+      when 'Skipped'
+        finding['status'] = 'skipped'
+      else 
+        finding['status'] = 'skipped'
+      end
+      [finding]
+    end
+
+    def impact(severity)
+      IMPACT_MAPPING[severity.to_sym]
+    end
+
+    # DBProtect report could have multiple issue entries for multiple findings of same issue type.
+    # The meta data is identical across entries 
+    # method collapse_duplicates return unique controls with applicable findings collapsed into it.
+    def collapse_duplicates(controls)
+      unique_controls = []
+
+      controls.map { |x| x['id'] }.uniq.each do |id|
+        collapsed_results = controls.select { |x| x['id'].eql?(id) }.map {|x| x['results']}
+        unique_control = controls.find { |x| x['id'].eql?(id) }
+        unique_control['results'] = collapsed_results.flatten
+        unique_controls << unique_control
+      end
+      unique_controls
+    end
+
+
+  end
+end

data/lib/heimdall_tools/fortify_mapper.rb

@@ -3,6 +3,7 @@ require 'heimdall_tools/hdf'
 require 'utilities/xml_to_hash'
 
 NIST_REFERENCE_NAME = 'Standards Mapping - NIST Special Publication 800-53 Revision 4'.freeze
+DEFAULT_NIST_TAG = ["SA-11", "RA-5"].freeze
 
 module HeimdallTools
   class FortifyMapper
@@ -68,7 +69,7 @@ module HeimdallTools
       references = rule['References']['Reference']
       references = [references] unless references.is_a?(Array)
       tag = references.detect { |x| x['Author'].eql?(NIST_REFERENCE_NAME) }
-      tag.nil? ? 'unmapped' : tag['Title'].match(/[a-zA-Z][a-zA-Z]-\d{1,2}/)
+      tag.nil? ? DEFAULT_NIST_TAG : tag['Title'].match(/[a-zA-Z][a-zA-Z]-\d{1,2}/)
     end
 
     def impact(classid)

data/lib/heimdall_tools/help/dbprotect_mapper.md

@@ -0,0 +1,5 @@
+  dbprotect_mapper translates DBProtect report in `Check Results Details` format XML to HDF format JSON be viewed on Heimdall.
+  
+Examples:
+
+  heimdall_tools dbprotect_mapper -x check_results_details_report.xml -o db_protect_hdf.json

data/lib/heimdall_tools/help/jfrog_xray_mapper.md

@@ -0,0 +1,5 @@
+  jfrog_xray_mapper translates an JFrog Xray results JSON file into HDF format JSON to be viewable in Heimdall
+
+Examples:
+
+  heimdall_tools jfrog_xray_mapper -j xray_results.json -o output-file-name.json

data/lib/heimdall_tools/help/nikto_mapper.md

@@ -0,0 +1,7 @@
+  nikto_mapper translates an Nikto results JSON file into HDF format JSON to be viewable in Heimdall
+  
+  Note: Current this mapper only support single target Nikto Scans.
+
+Examples:
+
+  heimdall_tools nikto_mapper [OPTIONS] -x <nikto-results-json> -o <hdf-scan-results.json>

data/lib/heimdall_tools/help/snyk_mapper.md

@@ -0,0 +1,7 @@
+  snyk_mapper translates an Snyk results JSON file into HDF format json to be viewable in Heimdall
+  
+  A separate HDF JSON is generated for each project reported in the Snyk Report.
+
+Examples:
+
+  heimdall_tools snyk_mapper -j snyk_results.json -o output-file-prefix

data/lib/heimdall_tools/jfrog_xray_mapper.rb

@@ -0,0 +1,142 @@
+require 'json'
+require 'csv'
+require 'heimdall_tools/hdf'
+require 'utilities/xml_to_hash'
+
+RESOURCE_DIR = Pathname.new(__FILE__).join('../../data')
+
+CWE_NIST_MAPPING_FILE = File.join(RESOURCE_DIR, 'cwe-nist-mapping.csv')
+
+IMPACT_MAPPING = {
+  high: 0.7,
+  medium: 0.5,
+  low: 0.3,
+}.freeze
+
+DEFAULT_NIST_TAG = ["SA-11", "RA-5"].freeze
+
+# Loading spinner sign
+$spinner = Enumerator.new do |e|
+  loop do
+    e.yield '|'
+    e.yield '/'
+    e.yield '-'
+    e.yield '\\'
+  end
+end
+
+module HeimdallTools
+  class JfrogXrayMapper
+    def initialize(xray_json, name=nil, verbose = false)
+      @xray_json = xray_json
+      @verbose = verbose
+
+      begin
+        @cwe_nist_mapping = parse_mapper
+        @project = JSON.parse(xray_json)
+
+      rescue StandardError => e
+        raise "Invalid JFrog Xray JSON file provided Exception: #{e}"
+      end
+    end
+
+    def finding(vulnerability)
+      finding = {}
+      finding['status'] = 'failed'
+      finding['code_desc'] = []
+      finding['code_desc'] << "source_comp_id : #{vulnerability['source_comp_id'].to_s }"
+      finding['code_desc'] << "vulnerable_versions : #{vulnerability['component_versions']['vulnerable_versions'].to_s }"
+      finding['code_desc'] << "fixed_versions : #{vulnerability['component_versions']['fixed_versions'].to_s }"
+      finding['code_desc'] << "issue_type : #{vulnerability['issue_type'].to_s }"
+      finding['code_desc'] << "provider : #{vulnerability['provider'].to_s }"
+      finding['code_desc'] = finding['code_desc'].join("\n")
+      finding['run_time'] = NA_FLOAT
+
+      # Xray results does not profile scan timestamp; using current time to satisfy HDF format
+      finding['start_time'] = NA_STRING
+      [finding]
+    end
+
+    def nist_tag(cweid)
+      entries = @cwe_nist_mapping.select { |x| cweid.include? x[:cweid].to_s }
+      tags = entries.map { |x| x[:nistid] }
+      tags.empty? ? DEFAULT_NIST_TAG : tags.flatten.uniq
+    end
+
+    def parse_identifiers(vulnerability, ref)
+      # Extracting id number from reference style CWE-297
+      vulnerability['component_versions']['more_details']['cves'][0][ref.downcase].map { |e| e.split("#{ref}-")[1]  }
+      rescue
+        return []
+    end
+
+    def impact(severity)
+      IMPACT_MAPPING[severity.downcase.to_sym]
+    end
+
+    def parse_mapper
+      csv_data = CSV.read(CWE_NIST_MAPPING_FILE, **{ encoding: 'UTF-8',
+                                                   headers: true,
+                                                   header_converters: :symbol,
+                                                   converters: :all })
+      csv_data.map(&:to_hash)
+    end
+
+    def desc_tags(data, label)
+      { "data": data || NA_STRING, "label": label || NA_STRING }
+    end
+
+    # Xray report could have multiple vulnerability entries for multiple findings of same issue type.
+    # The meta data is identical across entries 
+    # method collapse_duplicates return unique controls with applicable findings collapsed into it.
+    def collapse_duplicates(controls)
+      unique_controls = []
+
+      controls.map { |x| x['id'] }.uniq.each do |id|
+        collapsed_results = controls.select { |x| x['id'].eql?(id) }.map {|x| x['results']}
+        unique_control = controls.find { |x| x['id'].eql?(id) }
+        unique_control['results'] = collapsed_results.flatten
+        unique_controls << unique_control
+      end
+      unique_controls
+    end
+
+    def to_hdf
+      controls = []
+      vulnerability_count = 0
+      @project['data'].uniq.each do | vulnerability |
+        printf("\rProcessing: %s", $spinner.next)
+        
+        vulnerability_count +=1
+        item = {}
+        item['tags']               = {}
+        item['descriptions']       = []
+        item['refs']               = NA_ARRAY
+        item['source_location']    = NA_HASH
+        item['descriptions']       = NA_ARRAY
+
+        # Xray JSONs might note have `id` fields populated. 
+        # If thats a case MD5 hash is used to collapse vulnerability findings of the same type.
+        item['id']                 = vulnerability['id'].empty? ? OpenSSL::Digest::MD5.digest(vulnerability['summary'].to_s).unpack("H*")[0].to_s : vulnerability['id']
+        item['title']              = vulnerability['summary'].to_s
+        item['desc']               = vulnerability['component_versions']['more_details']['description'].to_s
+        item['impact']             = impact(vulnerability['severity'].to_s) 
+        item['code']               = NA_STRING
+        item['results']            = finding(vulnerability)
+
+        item['tags']['nist']       = nist_tag( parse_identifiers( vulnerability, 'CWE') )
+        item['tags']['cweid']      = parse_identifiers( vulnerability, 'CWE')
+
+        controls << item
+      end
+
+      controls = collapse_duplicates(controls)
+      results = HeimdallDataFormat.new(profile_name: "JFrog Xray Scan",
+                                       version: NA_STRING,
+                                       title: "JFrog Xray Scan", 
+                                       summary: "Continuous Security and Universal Artifact Analysis",
+                                       controls: controls)
+      results.to_hdf
+    end
+  end
+end

data/lib/heimdall_tools/nikto_mapper.rb

@@ -0,0 +1,149 @@
+require 'json'
+require 'csv'
+require 'heimdall_tools/hdf'
+
+RESOURCE_DIR = Pathname.new(__FILE__).join('../../data')
+
+NIKTO_NIST_MAPPING_FILE = File.join(RESOURCE_DIR, 'nikto-nist-mapping.csv')
+
+IMPACT_MAPPING = {
+  high: 0.7,
+  medium: 0.5,
+  low: 0.3,
+}.freeze
+
+DEFAULT_NIST_TAG = ["SA-11", "RA-5"].freeze
+
+# Loading spinner sign
+$spinner = Enumerator.new do |e|
+  loop do
+    e.yield '|'
+    e.yield '/'
+    e.yield '-'
+    e.yield '\\'
+  end
+end
+
+module HeimdallTools
+  class NiktoMapper
+    def initialize(nikto_json, name=nil, verbose = false)
+      @nikto_json = nikto_json
+      @verbose = verbose
+
+      begin
+        @nikto_nist_mapping = parse_mapper
+      rescue StandardError => e
+        raise "Invalid Nikto to NIST mapping file: Exception: #{e}"
+      end
+
+        # TODO: Support Multi-target scan results
+        # Nikto multi-target scans generate invalid format JSONs
+        # Possible workaround to use https://stackoverflow.com/a/58209963/1670307
+
+      begin
+        @project = JSON.parse(nikto_json)
+      rescue StandardError => e
+        raise "Invalid Nikto JSON file provided\nNote: nikto_mapper does not support multi-target scan results\n\nException: #{e}"
+      end
+    end
+
+    def extract_scaninfo(project)
+      info = {}
+      begin
+        info['policy'] = 'Nikto Website Scanner'
+        info['version'] = NA_STRING
+        info['projectName'] = "Host: #{project['host']} Port: #{project['port']}"
+        info['summary'] = "Banner: #{project['banner']}"
+
+        info
+      rescue StandardError => e
+        raise "Error extracting project info from nikto JSON file provided Exception: #{e}"
+      end
+    end
+
+    def finding(vulnerability)
+      finding = {}
+      finding['status'] = 'failed'
+      finding['code_desc'] = "URL : #{vulnerability['url'].to_s } Method: #{vulnerability['method'].to_s}"
+      finding['run_time'] = NA_FLOAT
+      finding['start_time'] = NA_STRING
+      [finding]
+    end
+
+    def nist_tag(niktoid)
+      entries = @nikto_nist_mapping.select { |x| niktoid.eql?(x[:niktoid].to_s) }
+      tags = entries.map { |x| x[:nistid] }
+      tags.empty? ? DEFAULT_NIST_TAG : tags.flatten.uniq
+    end
+
+    def impact(severity)
+      IMPACT_MAPPING[severity.to_sym]
+    end
+
+    def parse_mapper
+      csv_data = CSV.read(NIKTO_NIST_MAPPING_FILE, **{ encoding: 'UTF-8',
+                                                   headers: true,
+                                                   header_converters: :symbol})
+      csv_data.map(&:to_hash)
+    end
+
+    def desc_tags(data, label)
+      { "data": data || NA_STRING, "label": label || NA_STRING }
+    end
+
+    # Nikto report could have multiple vulnerability entries for multiple findings of same issue type.  
+    # The meta data is identical across entries   
+    # method collapse_duplicates return unique controls with applicable findings collapsed into it. 
+    def collapse_duplicates(controls) 
+      unique_controls = []  
+
+      controls.map { |x| x['id'] }.uniq.each do |id|  
+        collapsed_results = controls.select { |x| x['id'].eql?(id) }.map {|x| x['results']} 
+        unique_control = controls.find { |x| x['id'].eql?(id) } 
+        unique_control['results'] = collapsed_results.flatten 
+        unique_controls << unique_control 
+      end 
+      unique_controls 
+    end 
+
+    def to_hdf
+      controls = []
+      @project['vulnerabilities'].each do | vulnerability |
+        printf("\rProcessing: %s", $spinner.next)
+
+        item = {}
+        item['tags']               = {}
+        item['descriptions']       = []
+        item['refs']               = NA_ARRAY
+        item['source_location']    = NA_HASH
+        item['descriptions']       = NA_ARRAY
+
+        item['title']              = vulnerability['msg'].to_s
+        item['id']                 = vulnerability['id'].to_s
+
+        # Nikto results JSON does not description fields
+        # Duplicating vulnerability msg field
+        item['desc']               = vulnerability['msg'].to_s
+
+        # Nitko does not provide finding severity; hard-coding severity to medium 
+        item['impact']             = impact('medium') 
+        item['code']               = NA_STRING
+        item['results']            = finding(vulnerability)
+        item['tags']['nist']       = nist_tag( vulnerability['id'].to_s )
+        item['tags']['ösvdb']      = vulnerability['OSVDB']
+
+        controls << item
+      end
+
+      controls = collapse_duplicates(controls)
+      scaninfo = extract_scaninfo(@project)
+      results = HeimdallDataFormat.new(profile_name: scaninfo['policy'],
+                                       version: scaninfo['version'],
+                                       title: "Nikto Target: #{scaninfo['projectName']}",
+                                       summary: "Banner: #{scaninfo['summary']}",
+                                       controls: controls,
+                                       target_id: scaninfo['projectName'])
+      results.to_hdf
+    end
+  end
+end