dradis-burp 3.12.0 → 3.13.0

data/lib/dradis-burp.rb

@@ -6,3 +6,5 @@ require 'dradis/plugins/burp'
 
 # Load supporting Burp classes
 require 'burp/issue'
+require 'burp/html/issue'
+require 'burp/xml/issue'

data/lib/dradis/plugins/burp.rb

@@ -7,5 +7,6 @@ end
 
 require 'dradis/plugins/burp/engine'
 require 'dradis/plugins/burp/field_processor'
-require 'dradis/plugins/burp/importer'
+require 'dradis/plugins/burp/html/importer'
 require 'dradis/plugins/burp/version'
+require 'dradis/plugins/burp/xml/importer'

data/lib/dradis/plugins/burp/engine.rb

@@ -5,10 +5,21 @@ module Dradis
         isolate_namespace Dradis::Plugins::Burp
 
         include ::Dradis::Plugins::Base
-        description 'Processes Burp Scanner XML output'
+        description 'Processes Burp Scanner output'
         provides :upload
+
+        # Because this plugin provides two export modules, we have to overwrite
+        # the default .uploaders() method.
+        #
+        # See:
+        #  Dradis::Plugins::Upload::Base in dradis-plugins
+        def self.uploaders
+          [
+            Dradis::Plugins::Burp::Html,
+            Dradis::Plugins::Burp::Xml
+          ]
+        end
       end
     end
   end
 end
-

data/lib/dradis/plugins/burp/field_processor.rb

@@ -4,7 +4,12 @@ module Dradis
       class FieldProcessor < Dradis::Plugins::Upload::FieldProcessor
 
         def post_initialize(args={})
-          @burp_object = ::Burp::Issue.new(data)
+          @burp_object =
+            if data.is_a?(Nokogiri::XML::Element)
+              ::Burp::Xml::Issue.new(data)
+            elsif data.is_a?(Nokogiri::XML::NodeSet)
+              ::Burp::Html::Issue.new(data)
+            end
         end
 
         def value(args={})
@@ -20,4 +25,3 @@ module Dradis
     end
   end
 end
-

data/lib/dradis/plugins/burp/gem_version.rb

@@ -8,7 +8,7 @@ module Dradis
 
       module VERSION
         MAJOR = 3
-        MINOR = 12
+        MINOR = 13
         TINY = 0
         PRE = nil
 

data/lib/dradis/plugins/burp/html/importer.rb

@@ -0,0 +1,144 @@
+module Dradis::Plugins::Burp
+  # This module knows how to parse Burp HTML format.
+  module Html
+    def self.meta
+      package = Dradis::Plugins::Burp
+      {
+        name:        package::Engine::plugin_name,
+        description: 'Upload Burp Scanner output file (.html)',
+        version:     package.version
+      }
+    end
+
+    class Importer < Dradis::Plugins::Upload::Importer
+      def initialize(args={})
+        args[:plugin] = Dradis::Plugins::Burp
+        super(args)
+      end
+
+      def import(params = {})
+        logger.info { 'Parsing Burp Scanner HTML output file...' }
+        @doc = Nokogiri::HTML(File.read(params[:file]))
+        logger.info { 'Done.' }
+
+        # Issue headers are like: <span class="BODH0" id="X">
+        issue_headers = @doc.xpath("//span[contains(@class, 'BODH0')]")
+
+        if issue_headers.count.zero?
+          error = "Document doesn't seem to be in the Burp Scanner HTML format."
+          logger.fatal { error }
+          content_service.create_note text: error
+          return false
+        end
+
+        issue_headers.each do |header|
+          issue_id = header.attr('id')
+          html     = extract_html_fragment_for(issue_id)
+          process_html_issue(html)
+        end
+
+        logger.info { 'Burp Scanner results successfully imported' }
+        true
+      end
+
+      def process_html_issue(html_issue)
+        header     = html_issue.first
+        title      = header.text.gsub(/^\d+\.\S/, '')
+        burp_id =
+          if (link = header.css('a').first)
+            link.attr('href')[/\/([0-9a-f]+)_.*/, 1].to_i(16)
+          else
+            title
+          end
+        issue_id   = html_issue.attr('id').value
+        issue_text =
+          template_service.process_template(
+            template: 'issue',
+            data: html_issue
+          )
+
+        logger.info { "Processing issue #{issue_id}: #{title}" }
+        issue = content_service.create_issue(text: issue_text, id: burp_id)
+
+        # Evidence headers are like:
+        #   <span class="BODH1" id="X.Y">
+        # where:
+        #   X is the issue index
+        #   Y is the evidence index
+        evidence_headers = html_issue.xpath(
+          "//span[contains(@class, 'BODH1') and starts-with(@id, '#{issue_id}.')]"
+        )
+
+        # If there are no evidence headers inside this issue, this is a
+        # "single evidence" case: our evidence html is the issue html itself
+        if evidence_headers.count.zero?
+          process_html_evidence(html_issue, issue)
+        else
+          evidence_headers.each do |header|
+            evidence_id = header.attr('id')
+            html = extract_html_fragment_for(evidence_id)
+            process_html_evidence(html, issue)
+          end
+        end
+      end
+
+      def process_html_evidence(html_evidence, issue)
+        evidence_id = html_evidence.attr('id').value
+        logger.info { "Processing evidence #{evidence_id}" }
+
+        host_td    = html_evidence.search("td:starts-with('Host:')").first
+        host_label = host_td.next_element.text.split('//').last
+        host       = content_service.create_node(label: host_label, type: :host)
+
+        evidence_text =
+          template_service.process_template(
+            template: 'html_evidence',
+            data: html_evidence
+          )
+
+        content_service.create_evidence(
+          issue: issue,
+          node: host,
+          content: evidence_text
+        )
+      end
+
+      # Html for an issue and evidence is not nested inside an html element.
+      #
+      # An issue is the html fragment from <span id="X"> (where X is a single
+      # integer number: 1, 2, 3...) until the next span like that or the end of
+      # the file.
+      #
+      # An evidence is the html fragment from <span id="X.Y"> (where X is the
+      # issue index and Y the evidence index: 1.1, 1.2,...,2.1, 2.2,...) until
+      # the next evidence span (id="X.Z"), the next issue span (id="Y"), or the
+      # end of the file.
+      #
+      # This method extracts all the html related to as specific issue id or
+      # evidence id.
+      def extract_html_fragment_for(id)
+        next_id = if /\d+\.\d+/ =~ id
+                    id_parts = id.split('.')
+                    "#{id_parts[0]}.#{id_parts[1].to_i + 1}"
+                  else
+                    id.to_i + 1
+                  end
+
+        start_element = @doc.xpath("//span[@id='#{id}']")
+        return nil if start_element.empty?
+
+        ending_element = @doc.xpath("//span[@id='#{next_id}']")
+        if ending_element.empty? && /\d+\.\d+/ =~ id
+          next_id = id.split('.')[0].to_i + 1
+          ending_element = @doc.xpath("//span[@id='#{next_id}']")
+        end
+
+        xpath = "//*[preceding-sibling::span[@id='#{id}']"
+        xpath += " and following-sibling::span[@id='#{next_id}']" unless ending_element.empty?
+        xpath += ']'
+
+        start_element + @doc.xpath(xpath)
+      end
+    end
+  end
+end

data/lib/dradis/plugins/burp/xml/importer.rb

@@ -0,0 +1,150 @@
+module Dradis::Plugins::Burp
+
+  # This module knows how to parse Burp XML format.
+  module Xml
+    def self.meta
+      package = Dradis::Plugins::Burp
+      {
+        name:        package::Engine::plugin_name,
+        description: 'Upload Burp Scanner output file (.xml)',
+        version:     package.version
+      }
+    end
+
+    class Importer < Dradis::Plugins::Upload::Importer
+      def initialize(args={})
+        args[:plugin] = Dradis::Plugins::Burp
+        super(args)
+      end
+      def import(params = {})
+        file_content = File.read(params[:file])
+
+        if file_content =~ /base64="false"/
+          error =  "Burp input contains HTTP request / response data that hasn't been Base64-encoded.\n"
+          error << 'Please re-export your scanner results making sure the Base-64 encode option is selected.'
+
+          logger.fatal{ error }
+          content_service.create_note text: error
+          return false
+        end
+
+        logger.info { 'Parsing Burp Scanner XML output file...' }
+        doc = Nokogiri::XML(file_content)
+        logger.info { 'Done.' }
+
+        if doc.root.name != 'issues'
+          error = "Document doesn't seem to be in the Burp Scanner XML format."
+          logger.fatal { error }
+          content_service.create_note text: error
+          return false
+        end
+
+        # This will be filled in by the Processor while iterating over the issues
+        @hosts         = []
+        @affected_host = nil
+        @issue_text    = nil
+        @evidence_text = nil
+
+        doc.xpath('issues/issue').each do |xml_issue|
+          process_issue(xml_issue)
+        end
+
+        logger.info { 'Burp Scanner results successfully imported' }
+        true
+      end
+
+      # Creates the Nodes/properties
+      def process_issue(xml_issue)
+        host_label = xml_issue.at('host')['ip']
+        host_label = xml_issue.at('host').text if host_label.empty?
+        affected_host = content_service.create_node(label: host_label, type: :host)
+        logger.info { "\taffects: #{host_label}" }
+
+        unless @hosts.include?(affected_host.label)
+          @hosts << affected_host.label
+          url = xml_issue.at('host').text
+          affected_host.set_property(:hostname, url)
+          affected_host.save
+        end
+
+        # Burp extensions don't follow the "unique type for every Issue" logic
+        # so we have to deal with them separately
+        burp_extension_type = '134217728'.freeze
+        if xml_issue.at('type').text.to_str == burp_extension_type
+          process_extension_issues(affected_host, xml_issue)
+        else
+          process_burp_issues(affected_host, xml_issue)
+        end
+      end
+
+      # If the Issues come from the Burp app, use the type as the plugin_ic
+      def process_burp_issues(affected_host, xml_issue)
+        issue_name = xml_issue.at('name').text
+        issue_type = xml_issue.at('type').text.to_i
+
+        logger.info { "Adding #{issue_name} (#{issue_type})" }
+
+        create_issue(
+          affected_host: affected_host,
+          id: issue_type,
+          xml_issue: xml_issue
+        )
+      end
+
+      # If the Issues come from a Burp extension (type = 134217728), then
+      # use the name (spaces removed) as the plugin_id
+      def process_extension_issues(affected_host, xml_issue)
+        ext_name = xml_issue.at('name').text
+        ext_name = ext_name.gsub!(" ", "")
+
+        logger.info { "Adding #{ext_name}" }
+
+        create_issue(
+          affected_host: affected_host,
+          id: ext_name,
+          xml_issue: xml_issue
+        )
+      end
+
+      def create_issue(affected_host:, id:, xml_issue:)
+        issue_text =
+          template_service.process_template(
+            template: 'issue',
+            data: xml_issue
+          )
+
+        if issue_text.include?(::Burp::INVALID_UTF_REPLACE)
+          logger.info do
+            "\tdetected invalid UTF-8 bytes in your issue. " \
+            "Replacing them with '#{::Burp::INVALID_UTF_REPLACE}'."
+          end
+        end
+
+        issue = content_service.create_issue(text: issue_text, id: id)
+
+        logger.info do
+          "\tadding evidence for this instance to #{affected_host.label}."
+        end
+
+        evidence_text =
+          template_service.process_template(
+            template: 'evidence',
+            data: xml_issue
+          )
+
+        if evidence_text.include?(::Burp::INVALID_UTF_REPLACE)
+          logger.info do
+            "\tdetected invalid UTF-8 bytes in your evidence. " \
+            "Replacing them with '#{::Burp::INVALID_UTF_REPLACE}'."
+          end
+        end
+
+        content_service.create_evidence(
+          issue: issue,
+          node: affected_host,
+          content: evidence_text
+        )
+      end
+    end
+  end
+end

data/lib/tasks/thorfile.rb

@@ -3,18 +3,27 @@ class BurpTasks < Thor
 
   namespace "dradis:plugins:burp"
 
-  desc "upload FILE", "upload Burp XML results"
+  desc "upload FILE", "upload Burp XML or HTML results"
   def upload(file_path)
     require 'config/environment'
 
     unless File.exists?(file_path)
       $stderr.puts "** the file [#{file_path}] does not exist"
-      exit -1
+      exit(-1)
     end
 
     detect_and_set_project_scope
 
-    importer = Dradis::Plugins::Burp::Importer.new(task_options)
+    importer =
+      if File.extname(file_path) == '.xml'
+        Dradis::Plugins::Burp::Xml::Importer.new(task_options)
+      elsif File.extname(file_path) == '.html'
+        Dradis::Plugins::Burp::Html::Importer.new(task_options)
+      else
+        $stderr.puts "** Unsupported file. Must be .xml or .html"
+        exit(-2)
+      end
+
     importer.import(file: file_path)
   end
 

data/spec/burp_upload_spec.rb

@@ -3,17 +3,17 @@ require 'ostruct'
 
 describe 'Burp upload plugin' do
 
-  describe Burp::Issue do
+  describe Burp::Xml::Issue do
     it "handles invalid utf-8 bytes" do
       doc = Nokogiri::XML(File.read('spec/fixtures/files/invalid-utf-issue.xml'))
       xml_issue = doc.xpath('issues/issue').first
-      issue = Burp::Issue.new(xml_issue)
+      issue = Burp::Xml::Issue.new(xml_issue)
 
       expect{ issue.request.encode('utf-8') }.to_not raise_error
     end
   end
 
-  describe "Importer" do
+  describe  Dradis::Plugins::Burp::Xml::Importer do
     before(:each) do
       # Stub template service
       templates_dir = File.expand_path('../../templates', __FILE__)
@@ -21,7 +21,7 @@ describe 'Burp upload plugin' do
       .to receive(:default_templates_dir).and_return(templates_dir)
 
       # Init services
-      plugin = Dradis::Plugins::Burp
+      plugin = Dradis::Plugins::Burp::Xml
 
       @content_service = Dradis::Plugins::ContentService::Base.new(
         logger: Logger.new(STDOUT),
@@ -113,4 +113,68 @@ describe 'Burp upload plugin' do
     end
 
   end
+
+  describe  Dradis::Plugins::Burp::Html::Importer do
+    before(:each) do
+      # Stub template service
+      templates_dir = File.expand_path('../../templates', __FILE__)
+      expect_any_instance_of(Dradis::Plugins::TemplateService)
+      .to receive(:default_templates_dir).and_return(templates_dir)
+
+      # Init services
+      plugin = Dradis::Plugins::Burp::Html
+
+      @content_service = Dradis::Plugins::ContentService::Base.new(
+        logger: Logger.new(STDOUT),
+        plugin: plugin
+      )
+
+      @importer = plugin::Importer.new(
+        content_service: @content_service,
+      )
+
+      # Stub dradis-plugins methods
+      #
+      # They return their argument hashes as objects mimicking
+      # Nodes, Issues, etc
+      allow(@content_service).to receive(:create_node) do |args|
+        obj = OpenStruct.new(args)
+        obj.define_singleton_method(:set_property) { |_, __| }
+        obj
+      end
+      allow(@content_service).to receive(:create_issue) do |args|
+        OpenStruct.new(args)
+      end
+      allow(@content_service).to receive(:create_evidence) do |args|
+        OpenStruct.new(args)
+      end
+    end
+
+    it "creates nodes, issues, and evidence as needed" do
+
+      # Host node
+      #
+      # create_node should be called once for each issue in the xml,
+      # but ContentService knows it's already created and NOOPs
+      expect(@content_service).to receive(:create_node)
+      .with(hash_including label: 'github.com/dradis/dradis-burp')
+      .exactly(1).times
+
+      # # create_issue should be called once for each issue in the xml
+      expect(@content_service).to receive(:create_issue) do |args|
+        expect(args[:text]).to include("#[Title]#\nStrict transport security not enforced")
+        expect(args[:id]).to eq(16777984)
+        OpenStruct.new(args)
+      end.once
+      expect(@content_service).to receive(:create_evidence) do |args|
+        expect(args[:content]).to include("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+        expect(args[:issue].text).to include("#[Title]#\nStrict transport security not enforced")
+        expect(args[:node].label).to eq("github.com/dradis/dradis-burp")
+      end.once
+
+      # Run the import
+      @importer.import(file: 'spec/fixtures/files/burp.html')
+    end
+
+  end
 end