nexpose_ticketing 0.5.0 → 0.8.0

data/lib/nexpose_ticketing/config/servicenow.config

@@ -1,18 +1,19 @@
----
-# This configuration file defines all the options necessary to support the helper.
-# Fields marked (M) are mandatory.
-#
-
-# (M) Helper class name
-:helper_name: ServiceNowHelper
-
-# (M) ServiceNow url (currently requires using JSON)
-:servicenow_url: https://your.service-now.com/u_rpd_vulnerabilities.do?JSON
-# (M) Username for ServiceNow
-:username: admin
-# (M) Password for above username
-:password: admin
-# (M) If 'Y', SSL connections will output to stderr (default is 'N')
-:verbose_mode: N
-# (M) Amount of times the helper will follow 301/302 redirections
-:redirect_limit: 10
+---
+# This configuration file defines all the options necessary to support the helper.
+# Fields marked (M) are mandatory.
+#
+
+# (M) Helper class name
+:helper_name: ServiceNowHelper
+
+# (M) ServiceNow url (currently requires using JSON)
+:servicenow_url: https://Servicenow_url/u_rpd_vulnerabilities.do?JSONv2
+# Use this URL if you have loaded the ticketing UpdateSet for ServiceNow
+# (M) Username for ServiceNow
+:username: admin
+# (M) Password for above username
+:password: nxadmin
+# (M) If 'Y', SSL connections will output to stderr (default is 'N')
+:verbose_mode: N
+# (M) Amount of times the helper will follow 301/302 redirections
+:redirect_limit: 10

data/lib/nexpose_ticketing/config/servicenow_updateset/Rapid7 Nexpose Ticketing.glide-calgary-02-15-2013__patch2-hotfix5.xml

@@ -305,7 +305,7 @@ Support Phone: (866) 390-8113</description><device_type>browser</device_type><hi
 <category>customer</category>
 <comments/>
 <name>dictionary_u_rpd_vulnerabilities_55952fb42c46425d84de8f4fda4ce339</name>
-<payload><![CDATA[<?xml version="1.0" encoding="UTF-8"?><record_update><database><element extends="sys_import_set_row" label="Vulnerabilities" name="u_rpd_vulnerabilities" type="collection"><element label="work_notes" max_length="40" name="u_work_notes" type="string"/><element label="urgency" max_length="40" name="u_urgency" type="integer"/><element label="short_description" max_length="40" name="u_short_description" type="string"/><element label="category" max_length="40" name="u_category" type="string"/><element label="impact" max_length="40" name="u_impact" type="integer"/><element label="caller_id" max_length="40" name="u_caller_id" type="string"/></element></database><sys_app_file action="INSERT_OR_UPDATE"><sys_app/><sys_code>!!1#D/</sys_code><sys_created_by>admin</sys_created_by><sys_created_on>2014-04-02 20:14:25</sys_created_on><sys_id>221dc59634735140df776b3ac83c75c2</sys_id><sys_mod_count>0</sys_mod_count><sys_name>u_rpd_vulnerabilities</sys_name><sys_parent display_value="Rpd Vulnerabilities">2a1dc59634735140df776b3ac83c75ae</sys_parent><sys_path>!!1#C/!!1#D/</sys_path><sys_policy/><sys_source_deleted>false</sys_source_deleted><sys_source_id element="NULL" name="u_rpd_vulnerabilities" sys_source_table="sys_dictionary">ee1dc59634735140df776b3ac83c75c0</sys_source_id><sys_source_table>sys_dictionary</sys_source_table><sys_type>code</sys_type><sys_update_name>dictionary_u_rpd_vulnerabilities_55952fb42c46425d84de8f4fda4ce339</sys_update_name><sys_updated_by>admin</sys_updated_by><sys_updated_on>2014-04-08 18:04:59</sys_updated_on></sys_app_file></record_update>]]></payload>
+<payload><![CDATA[<?xml version="1.0" encoding="UTF-8"?><record_update><database><element extends="sys_import_set_row" label="Vulnerabilities" name="u_rpd_vulnerabilities" type="collection"><element label="work_notes" max_length="50000" name="u_work_notes" type="string"/><element label="urgency" max_length="40" name="u_urgency" type="integer"/><element label="short_description" max_length="40" name="u_short_description" type="string"/><element label="category" max_length="40" name="u_category" type="string"/><element label="impact" max_length="40" name="u_impact" type="integer"/><element label="caller_id" max_length="40" name="u_caller_id" type="string"/></element></database><sys_app_file action="INSERT_OR_UPDATE"><sys_app/><sys_code>!!1#D/</sys_code><sys_created_by>admin</sys_created_by><sys_created_on>2014-04-02 20:14:25</sys_created_on><sys_id>221dc59634735140df776b3ac83c75c2</sys_id><sys_mod_count>0</sys_mod_count><sys_name>u_rpd_vulnerabilities</sys_name><sys_parent display_value="Rpd Vulnerabilities">2a1dc59634735140df776b3ac83c75ae</sys_parent><sys_path>!!1#C/!!1#D/</sys_path><sys_policy/><sys_source_deleted>false</sys_source_deleted><sys_source_id element="NULL" name="u_rpd_vulnerabilities" sys_source_table="sys_dictionary">ee1dc59634735140df776b3ac83c75c0</sys_source_id><sys_source_table>sys_dictionary</sys_source_table><sys_type>code</sys_type><sys_update_name>dictionary_u_rpd_vulnerabilities_55952fb42c46425d84de8f4fda4ce339</sys_update_name><sys_updated_by>admin</sys_updated_by><sys_updated_on>2014-04-08 18:04:59</sys_updated_on></sys_app_file></record_update>]]></payload>
 <remote_update_set display_value="Rapid7 Nexpose Ticketing">1fe9af88340c6140df776b3ac83c7535</remote_update_set>
 <replace_on_upgrade>false</replace_on_upgrade>
 <sys_created_by>admin</sys_created_by>

data/lib/nexpose_ticketing/config/ticket_service.config

@@ -21,7 +21,7 @@
   # Timeout in seconds. The number of seconds the GEM waits for a response from Nexpose before exiting.
   :timeout: 10800
   # Ticket batching. Breaks ticket processing into groups of value size controlling resource utilisation of both systems.
-  :batch_size: 300
+  :batch_size: 200
   # (M) For 'I' & 'V' mode. Set to 'Y' for tickets that have been fixed to be closed after update, set to 'N' for ticket to be left
   # open for a user to manually close or change the status.
   :close_old_tickets_on_update: Y
@@ -34,8 +34,8 @@
 # Nexpose options.
 :nexpose_data:
   # (M) Nexpose console hostname.
-  :nxconsole: 127.0.0.1 
+  :nxconsole: nexpose-console.host.com
   # (M) Nexpose username.
-  :nxuser: nxadmin
+  :nxuser: admin
   # (M) Nexpose password.
-  :nxpasswd: nxadmin
+  :nxpasswd: admin

data/lib/nexpose_ticketing/helpers/jira_helper.rb

@@ -3,17 +3,82 @@ require 'net/http'
 require 'net/https'
 require 'uri'
 require 'csv'
+
 # This class serves as the JIRA interface
 # that creates issues within JIRA from vulnerabilities
 # found in Nexpose.
 # Copyright:: Copyright (c) 2014 Rapid7, LLC.
 class JiraHelper
+  # TODO: Add V Mode.
+  # TODO: Allow updates/closed loop.
   attr_accessor :jira_data, :options
   def initialize(jira_data, options)
     @jira_data = jira_data
     @options = options
   end
 
+  # Generates the NXID. The NXID is a unique indentifier used to find and update and/or close tickets. 
+  #
+  # * *Args*    :
+  #   - +site_id+ -  Site ID the tickets are being generated for. Required for all ticketing modes
+  #   - +row+ -  Row from the generated Nexpose CSV report. Required for default ('D') mode.
+  #   - +current_ip+ -  The IP address of that this ticket is for. Required for IP mode ('I') mode.
+  #
+  # * *Returns* :
+  #   - NXID string.
+  #
+  def generate_nxid(site_id, row=nil, current_ip=nil)
+    fail 'Site ID is required to generate the NXID.' if site_id.empty?
+    case @options[:ticket_mode]
+      # 'D' Default mode: IP *-* Vulnerability
+      when 'D'
+        fail 'Row is required to generate the NXID in \'D\' mode.' if row.nil? || row.empty?
+        @nxid = "#{site_id}#{row['asset_id']}#{row['vulnerability_id']}#{row['solution_id']}"
+      # 'I' IP address mode: IP address -* Vulnerability
+      when 'I'
+        fail 'Current IP is required to generate the NXID in \'I\' mode.' if current_ip.nil? || current_ip.empty?
+        @nxid = "#{site_id}#{current_ip.tr('.','')}"
+      # 'V' mode net yet implemented.
+      # 'V' Vulnerability mode: Vulnerability -* IP address
+      #          when 'V'
+      #            @NXID = "#{site_id}#{row['current_asset_id']}#{row['current_vuln_id']}"
+      else
+        fail 'Could not close tickets - do not understand the ticketing mode!'
+    end
+  end
+
+  # Fetches the Jira ticket key e.g INT-1. This is required to post updates to the Jira.
+  #
+  # * *Args*    :
+  #   - +JQL query string+ -  Jira's Query Language string used to search for a ticket key.
+  #
+  # * *Returns* :
+  #   - Jira ticket key if found, nil otherwise.
+  #
+  def get_jira_key(jql_query)
+    fail 'JQL query string cannot be empty.' if jql_query.empty?
+    headers = { 'Content-Type' => 'application/json',
+                'Accept' => 'application/json' }
+
+    url = URI.parse(("#{@jira_data[:jira_url]}".split("/")[0..-2].join("/") + '/search'))
+    url.query = [url.query, URI.escape(jql_query)].compact.join('&')
+    req = Net::HTTP::Get.new(url.to_s, headers)
+    req.basic_auth @jira_data[:username], @jira_data[:password]
+    resp = Net::HTTP.new(url.host, url.port)
+
+    # Enable this line for debugging the https call.
+    # resp.set_debug_output $stderr
+
+    resp.use_ssl = true if @jira_data[:jira_url].to_s.start_with?('https')
+    resp.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    response = resp.request(req)
+
+    issues = JSON.parse(response.body)['issues']
+    #TODO: fail(?) if more than one issue exits with a "unique" NXID
+    return nil if issues.nil? || !issues.any? || issues.size > 1
+    return issues[0]['key']
+  end
+
   def create_tickets(tickets)
     fail 'Ticket(s) cannot be empty.' if tickets.empty? || tickets.nil?
     tickets.each do |ticket|
@@ -24,8 +89,10 @@ class JiraHelper
       req.basic_auth @jira_data[:username], @jira_data[:password]
       req.body = ticket
       resp = Net::HTTP.new(url.host, url.port)
+
       # Enable this line for debugging the https call.
-      # resp.set_debug_output $stderr
+      #resp.set_debug_output $stderr
+
       resp.use_ssl = true if @jira_data[:jira_url].to_s.start_with?('https')
       resp.verify_mode = OpenSSL::SSL::VERIFY_NONE
       resp.start { |http| http.request(req) }
@@ -33,6 +100,7 @@ class JiraHelper
   end
 
   # Prepares tickets from the CSV.
+  # TODO Implement V Version.
   def prepare_create_tickets(vulnerability_list, site_id)
     @ticket = Hash.new(-1)
     case @options[:ticket_mode]
@@ -50,7 +118,7 @@ class JiraHelper
   # Prepares and creates tickets in default mode.
   def prepare_tickets_default(vulnerability_list, site_id)
     tickets = []
-    CSV.parse(vulnerability_list.chomp, headers: :first_row)  do |row|
+    CSV.parse(vulnerability_list.chomp, headers: :first_row) do |row|
       # JiraHelper doesn't like new line characters in their summaries.
       summary = row['summary'].gsub(/\n/, ' ')
       ticket = {
@@ -58,7 +126,7 @@ class JiraHelper
               'project' => {
                   'key' => "#{@jira_data[:project]}" },
               'summary' => "#{row['ip_address']} => #{summary}",
-              'description' => "#{row['fix']} \n\n #{row['url']}",
+              'description' => "CVSS Score: #{row['cvss_score']} \n\n #{row['fix']} \n\n #{row['url']} \n\n\n NXID: #{generate_nxid(site_id, row)}",
               'issuetype' => {
                   'name' => 'Task' }
           }
@@ -68,11 +136,19 @@ class JiraHelper
     tickets
   end
 
-  # Prepares and creates tickets in IP mode.
+  # Prepare tickets from the CSV of vulnerabilities exported from Nexpose. This method batches tickets 
+  # per IP i.e. any vulnerabilities for a single IP in one ticket
+  #
+  #   - +vulnerability_list+ -  CSV of vulnerabilities within Nexpose.
+  #   - +site_id+ -  Site ID the vulnerability list was generate from.
+  #
+  # * *Returns* :
+  #   - List of JSON-formated tickets for updating within Jira.
+  #
   def prepare_tickets_by_ip(vulnerability_list, site_id)
     tickets = []
     current_ip = -1
-    CSV.parse(vulnerability_list.chomp, headers: :first_row)  do |row|
+    CSV.parse(vulnerability_list.chomp, headers: :first_row) do |row|
       if current_ip == -1
         current_ip = row['ip_address']
         @ticket = {
@@ -86,19 +162,161 @@ class JiraHelper
             }
         }
       end
+      # TODO: Better formating this.
       if current_ip == row['ip_address']
-        @ticket['fields']['description'] += "\n ==============================\n
-          #{row['summary']} \n ==============================\n
-          \n #{row['fix']}\n\n #{row['url']}"
+        @ticket['fields']['description'] +=
+        "\n ==============================\n\n
+        #{row['summary']} \n CVSS Score: #{row['cvss_score']}
+        \n\n ==============================\n
+        \n Source: #{row['source']}, Reference: #{row['reference']}
+        \n
+        \n First seen: #{row['first_discovered']}
+        \n Last seen: #{row['most_recently_discovered']}
+        \n Fix:
+        \n #{row['fix']}\n\n #{row['url']}
+        \n
+        \n\n"
       end
       unless current_ip == row['ip_address']
-        @ticket = @ticket.to_json
-        tickets.push(@ticket)
+        @ticket['fields']['description'] += "\n\n\n NXID: #{generate_nxid(site_id, row, current_ip)}"
+        tickets.push(@ticket.to_json)
         current_ip = -1
         redo
       end
     end
+    @ticket['fields']['description'] += "\n\n\n NXID: #{generate_nxid(site_id, nil, current_ip)}"
     tickets.push(@ticket.to_json) unless @ticket.nil?
     tickets
   end
+
+  # Sends ticket closure (in JSON format) to Jira individually (each ticket in the list
+  # as a separate web service call).
+  #
+  # * *Args*    :
+  #   - +tickets+ -  List of Jira ticket Keys to be closed.
+  #
+  def close_tickets(tickets)
+    if tickets.nil? || tickets.empty?
+      #@log.log_message("No tickets to close.")
+      #TODO: Log error
+    else
+      headers = { 'Content-Type' => 'application/json',
+                  'Accept' => 'application/json' }
+
+      tickets.each { |ticket|
+        url = URI.parse(("#{@jira_data[:jira_url]}#{ticket}/transitions"))
+        req = Net::HTTP::Post.new(url.to_s, headers)
+        req.basic_auth @jira_data[:username], @jira_data[:password]
+
+        #TODO: May need to make this customisable as Jira does not allow for some transitions depending on workflow.
+        req.body = {"transition" => {"id" => "2"}, "fields" => {"resolution" => {"name" => "Fixed"}}}.to_json
+
+        resp = Net::HTTP.new(url.host, url.port)
+
+        # Enable this line for debugging the https call.
+        #resp.set_debug_output $stderr
+
+        resp.use_ssl = true
+        resp.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        resp.start { |http| http.request(req) }
+      }
+    end
+  end
+
+  # Prepare ticket closures from the CSV of vulnerabilities exported from Nexpose.
+  #
+  # * *Args*    :
+  #   - +vulnerability_list+ -  CSV of vulnerabilities within Nexpose.
+  #
+  # * *Returns* :
+  #   - List of Jira ticket Keys to be closed.
+  #
+  def prepare_close_tickets(vulnerability_list, site_id)
+    @nxid = nil
+    tickets = []
+    CSV.parse(vulnerability_list.chomp, headers: :first_row)  do |row|
+      @nxid = generate_nxid(site_id, nil, row['ip_address'])
+      # Query Jira for the ticket by unique id (generated NXID)
+      queried_key = get_jira_key("jql=project=#{@jira_data[:project]} AND description ~ \"NXID: #{@nxid}\" AND (status != Closed AND status != Resolved)&fields=key")
+      if queried_key.nil? || queried_key.empty?
+        #TODO: Log error
+      else
+        #Jira uses a post call to the ticket key path to close the ticket. The "prepared batch of tickets" in this case is just a collection Jira ticket key's to close.
+        tickets.push(queried_key)
+      end
+    end
+    tickets
+  end
+
+  # Sends ticket updates (in JSON format) to Jira individually (each ticket in the list as a
+  # separate HTTP post).
+  #
+  # * *Args*    :
+  #   - +tickets+ -  List of JSON-formatted ticket updates.
+  #
+  def update_tickets(tickets)
+    if (tickets.nil? || tickets.empty?) then
+      #@log.log_message('No tickets to update.')
+	  #TODO: Log error
+    else
+      tickets.each do |ticket_details|
+        headers = {'Content-Type' => 'application/json',
+                   'Accept' => 'application/json'}
+
+        (ticket_details.first.nil?) ? send_whole_ticket = true : send_whole_ticket = false
+
+        url = "#{jira_data[:jira_url]}"
+        url += "#{ticket_details.first}" unless send_whole_ticket
+        uri = URI.parse(url)
+
+        send_whole_ticket ? req = Net::HTTP::Post.new(uri.to_s, headers) : req = Net::HTTP::Put.new(uri.to_s, headers)
+
+        req.basic_auth @jira_data[:username], @jira_data[:password]
+        send_whole_ticket ?
+            req.body = ticket_details.last :
+            req.body = {'update' => {'description' => [{'set' => "#{JSON.parse(ticket_details[1])['fields']['description']}"}]}}.to_json
+
+        resp = Net::HTTP.new(uri.host, uri.port)
+
+        # Enable this line for debugging the https call.
+        #resp.set_debug_output $stderr
+
+        resp.use_ssl = true if uri.to_s.start_with?('https')
+        resp.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        resp.start { |http| http.request(req) }
+      end
+    end
+  end
+
+  # Prepare ticket updates from the CSV of vulnerabilities exported from Nexpose. This method batches tickets 
+  # per IP i.e. any vulnerabilities for a single IP in one ticket
+  #
+  #   - +vulnerability_list+ -  CSV of vulnerabilities within Nexpose.
+  #   - +site_id+ -  Site ID the vulnerability list was generate from.
+  #
+  # * *Returns* :
+  #   - List of JSON-formated tickets for updating within Jira.
+  #
+  def prepare_update_tickets(vulnerability_list, site_id)
+    fail 'Ticket updates are only supported in IP-address mode.' if @options[:ticket_mode] != 'I'
+    #Jira uses the ticket key to push updates. Since new IPs won't have a Jira key, generate new tickets for all of the IPs found.
+    updated_tickets = prepare_tickets_by_ip(vulnerability_list, site_id)
+    tickets_to_send = []
+
+    #Find the keys that exist (IPs that have tickets already)
+    updated_tickets.each do |ticket|
+      nxid = JSON.parse(ticket)['fields']['description'].squeeze("\n").lines.to_a.last
+      if (nxid.slice! "NXID:").nil?
+        #TODO - log error.
+        #Could not get NXID from the last line in the description. Do not push the invalid description.
+        next
+      end
+      queried_key = get_jira_key("jql=project=#{@jira_data[:project]} AND description ~ \"NXID: #{nxid.strip}\" AND (status != Closed AND status != Resolved)&fields=key")
+      ticket_key_pair = []
+      ticket_key_pair << queried_key
+      ticket_key_pair << ticket
+      tickets_to_send << ticket_key_pair
+    end
+    tickets_to_send
+  end
 end

data/lib/nexpose_ticketing/helpers/servicedesk_helper.rb

@@ -1,5 +1,6 @@
 require 'net/http'
 require 'nokogiri'
+require 'dbm'
 
 class ServiceDeskHelper
     attr_accessor :servicedesk_data, :options, :log
@@ -108,8 +109,6 @@ class ServiceDeskHelper
                 }
             }
         end
-        puts request.to_xml
-
         return request.to_xml
     end
 
@@ -163,7 +162,7 @@ class ServiceDeskHelper
         tickets = []
         hostVulns = {}
         CSV.parse( vulnerability_list.chomp, headers: :first_row )  do |vuln|
-            hostVulns["#{site_id}#{vuln['ip_address']}"] = { :ip => vuln['ip_address'], :description => "" } if not hostVulns.has_key?(vuln['asset_id'])
+            hostVulns["#{site_id}#{vuln['ip_address']}"] = { :ip => vuln['ip_address'], :description => "" } if not hostVulns.has_key?("#{site_id}#{vuln['ip_address']}")
             hostVulns["#{site_id}#{vuln['ip_address']}"][:description] += "Summary: #{vuln['summary']}\nFix: #{vuln['fix']}\nURL: #{vuln['url']}\n\n"
         end
 
@@ -184,14 +183,24 @@ class ServiceDeskHelper
                             'INPUT_DATA' => ticket[:description] )
 
         response = Nokogiri::XML.parse( res.read_body )
-        status = Integer(response.xpath('//statuscode').text)
+        begin
+            status = response.xpath('//statuscode').text
+            if status.empty?
+                status_code = -1
+            else
+                status_code = Integer( status )
+            end
         
-        if status != 200
-            @log.log_message("Unable to create ticket #{ticket}, got response #{response.to_xml}")
-            return
-        end
+            if status_code != 200
+                @log.log_message("Unable to create ticket #{ticket}, got response #{response.to_xml}")
+                return
+            end
 
-        workorderid = Integer(response.xpath('//workorderid').text)
+            workorderid = Integer(response.xpath('//workorderid').text)
+        rescue ArgumentError => ae
+            @log.log_message("Failed to parse response from servicedesk #{response}")
+            raise ae
+        end
 
         @log.log_message( "created ticket #{workorderid}")
         add_ticket_to_database( workorderid, ticket[:nxid] )

data/lib/nexpose_ticketing/helpers/servicenow_helper.rb

@@ -8,7 +8,7 @@ require 'nexpose_ticketing/nx_logger'
 # Serves as the ServiceNow interface for creating/updating issues from 
 # vulnelrabilities found in Nexpose.
 class ServiceNowHelper
-  attr_accessor :servicenow_data, :options, :log
+  attr_accessor :servicenow_data, :options, :log, :transform
   def initialize(servicenow_data, options)
     @servicenow_data = servicenow_data
     @options = options
@@ -124,7 +124,7 @@ class ServiceNowHelper
 
   # Prepares a list of vulnerabilities into a list of JSON-formatted tickets (incidents) for 
   # ServiceNow. The preparation by default means that each vulnerability within Nexpose is a 
-  # separate incident within ServiceNow.  This makes for smaller, more actionalble incidents but 
+  # separate incident within ServiceNow.  This makes for smaller, more actionalble incidents but
   # could lead to a very large total number of incidents.
   #
   # * *Args*    :
@@ -143,6 +143,7 @@ class ServiceNowHelper
       @log.log_message("Creating ticket with IP address: #{row['ip_address']}, site id: #{site_id} and summary: #{summary}")
       # NXID in the u_work_notes is a unique identifier used to query incidents to update/resolve 
       # incidents as they are resolved in Nexpose.
+
       ticket = {
           'sysparm_action' => 'insert',
           'u_caller_id' => "#{@servicenow_data[:username]}",