nexpose_servicenow 0.4.24 → 0.5.1

data/lib/nexpose_servicenow/nexpose_helper.rb

@@ -7,12 +7,12 @@ module NexposeServiceNow
   class NexposeHelper
     def initialize(url, port, username, password)
       @log = NexposeServiceNow::NxLogger.instance
-    	@url = url
-    	@port = port
+      @url = url
+      @port = port
       @username = username
       @password = password
 
-    	@nsc = connect(username, password)
+      @nsc = connect(username, password)
 
       @timeout = 7200
     end
@@ -22,7 +22,7 @@ module NexposeServiceNow
         return [ id: -1, report_name: get_report_name(query_name) ]
       end
 
-      ids.map { |id| { id: id, report_name: get_report_name(query_name, id) }}
+      ids.map { |id| { id: id, report_name: get_report_name(query_name, id) } }
     end
 
     def self.get_report_name(query_name, id=nil)
@@ -37,36 +37,60 @@ module NexposeServiceNow
     end
 
     def create_report(query_name, ids, id_type, output_dir, query_options={})
-    	output_dir = File.expand_path(output_dir.to_s)
+      output_dir = File.expand_path(output_dir.to_s)
 
       #A single report doesn't use site filters        
       ids = [-1] unless Queries.multiple_reports?(query_name)
 
+      #If running latest_scans, send up information
+      if query_name == 'latest_scans'
+        @log.on_connect(@url, @port, @nsc.session_id, '{}')
+      end
+
       reports = []
       ids.each do |id|
         report_name = self.class.get_report_name(query_name, id)
         clean_up_reports(report_name)
 
-        delta_options = create_query_options(query_options, id)
-        delta_options['site_id'] = id
+        # Should use the value from historical_data.rb
+        default_delta = if id_type.to_s == 'asset_group'
+                          '1985-01-01 00:00:00'
+                        elsif id_type.to_s == 'site'
+                          0
+                        end
+
+        delta_options = create_query_options(query_options, id, default_delta)
+
+        # TODO: Refactor this into 'create_query_options'
+        delta_options[:site_id] = id
+        delta_options[:id_type] = id_type.to_s
+        delta_options[:filters] = query_options[:filters] || {}
+
+        min_cvss = if delta_options[:filters][:cvss].nil?
+                     0
+                   else
+                     delta_options[:filters][:cvss].first
+                   end
 
         query = Queries.send(query_name, delta_options)
-        report_id = generate_config(query, report_name, [id], id_type)
+
+        report_id = generate_config(query, report_name, [id], id_type, min_cvss)
 
         run_report(report_id)
         reports << save_report(report_name, report_id, output_dir)
       end
 
-      return reports
+      reports
     end
 
-    def create_query_options(query_options, nexpose_id=nil)
+    def create_query_options(query_options, nexpose_id=nil, default=0)
       options = {}
       options[:vuln_query_date] = query_options[:vuln_query_date]
 
-      return options if nexpose_id == nil || nexpose_id == -1
-      return 0 if query_options[:last_scans].empty?
-      options[:last_scan_id] = "#{query_options[:last_scans][nexpose_id] || 0}"
+      return options if nexpose_id.nil? || nexpose_id == -1
+      return 0 if query_options[:delta_values].empty?
+
+      options[:delta] = "#{query_options[:delta_values][nexpose_id] || default}"
 
       @log.log_message("Query options: #{options}")
       
@@ -74,23 +98,23 @@ module NexposeServiceNow
     end
 
     def clean_up_reports(report_name)
-    	#log 'Deleting existing report...'
       reports = @nsc.list_reports
-		  reports.select! { |r| r.name.start_with? report_name }
-		  reports.each do |r|
-		    @nsc.delete_report_config(r.config_id)
-		  end
-		end
+      reports.select! { |r| r.name.start_with? report_name }
+      reports.each { |r| @nsc.delete_report_config(r.config_id) }
+    end
 
-    def generate_config(query, report_name, ids, id_type)
+    def generate_config(query, report_name, ids, id_type, min_severity=0)
       @nsc = connect(@username, @password)
-      @log.log_message "Generating query config with name #{report_name}..."
-    	report_config = Nexpose::ReportConfig.new(report_name, nil, 'sql')
+      @log.log_message "Generating report config with name #{report_name}..."
+      report_config = Nexpose::ReportConfig.new(report_name, nil, 'sql')
       report_config.add_filter('version', '2.0.1')
       report_config.add_filter('query', query)
 
+      id_type = id_type.to_s.split('_').last
+      ids.each { |id| report_config.add_filter(id_type, id) unless id == -1 }
+
+
       @log.log_message "Saving report config #{report_name}..."
-      ids.each { |id| report_config.add_filter(id_type, id) if id != -1 }
       report_id = report_config.save(@nsc, false)
       @log.log_message "Report #{report_name} saved"
 
@@ -98,13 +122,13 @@ module NexposeServiceNow
     end
 
     def run_report(report_id)
-    	@log.log_message "Running report #{report_id}..."
+      @log.log_message "Running report #{report_id}..."
       @nsc.generate_report(report_id, false)
       wait_for_report(report_id)
     end
 
     def wait_for_report(id)
-      wait_until(:fail_on_exceptions => TRUE, :on_timeout => "Report generation timed out. Status: #{r = @nsc.last_report(id); r ? r.status : 'unknown'}") {
+      wait_until(:fail_on_exceptions => true, :on_timeout => "Report generation timed out. Status: #{r = @nsc.last_report(id); r ? r.status : 'unknown'}") {
         if %w(Failed Aborted Unknown).include?(@nsc.last_report(id).status)
           raise "Report failed to generate! Status <#{@nsc.last_report(id).status}>"
         end
@@ -132,13 +156,13 @@ module NexposeServiceNow
     end
 
     def save_report(report_name, report_id, output_dir)
-    	@log.log_message 'Saving report...'
-    	local_file_name = self.class.get_filepath(report_name, output_dir)
-		  File.delete(local_file_name) if File.exists? local_file_name
+      @log.log_message 'Saving report...'
+      local_file_name = self.class.get_filepath(report_name, output_dir)
+      File.delete(local_file_name) if File.exists? local_file_name
 
-		  #log 'Downloading report...'
-		  report_details = @nsc.last_report(report_id)
-		  File.open(local_file_name, 'wb') do |f| 
+      #log 'Downloading report...'
+      report_details = @nsc.last_report(report_id)
+      File.open(local_file_name, 'wb') do |f| 
         f.write(@nsc.download(report_details.uri))
       end
       
@@ -146,35 +170,33 @@ module NexposeServiceNow
         # Refresh the connection
         @nsc = connect(@username, @password)
 
-		    #Got the report, cleanup server-side
-		    @nsc.delete_report_config(report_id)
+        #Got the report, cleanup server-side
+        @nsc.delete_report_config(report_id)
       rescue
-        @log.log_error_message "Error deleting report"
+        @log.log_error_message 'Error deleting report'
       end
 
       local_file_name
     end
 
-  	def connect(username, password)
-  		begin
-		    console = Nexpose::Connection.new(@url, username, password)
-		    console.login
-		    @log.log_message 'Logged in.'
-		  rescue Exception => e
-		    @log.log_error_message 'Error logging in...'
-		    @log.log_error_message e
-
-        $stderr.puts "ERROR: Could not log in. Check log and Nexpose settings.\n#{e}"
-		    exit -1
-		  end
-
-      #@log.on_connect(@url, @port || 3780, console.session_id, "{}")
-		  console
-  	end
-
-    def all_sites
-      @nsc.sites.map { |s| s.id }
+    def connect(username, password)
+      begin
+        connection = Nexpose::Connection.new(@url, username, password, @port)
+        connection.login
+        @log.log_message 'Logged in.'
+      rescue Exception => e
+        msg = "ERROR: Could not log in. Check log and settings.\n#{e}"
+        @log.log_error_message msg
+        $stderr.puts msg
+        exit -1
+      end
+
+      connection
     end
 
+    # Pulls the collection IDs from Nexpose (e.g. asset groups, sites)
+    def collection_ids(collection_type)
+      @nsc.send("#{collection_type}s").map { |s| s.id }.sort
+    end
   end
 end

data/lib/nexpose_servicenow/queries.rb

@@ -98,8 +98,8 @@ module NexposeServiceNow
                 FROM dim_vulnerability_category dvc) dvc USING (vulnerability_id)
         WHERE date_modified >= '#{options[:vuln_query_date]}'"
     end
-
-    #Filter by site.
+    
+    # Filter by site.
     def self.assets(options={})
       "SELECT coalesce(host_name, CAST(dim_asset.asset_id as text)) as Name,
         dim_asset.ip_address,
@@ -116,7 +116,6 @@ module NexposeServiceNow
         fact_asset.pci_status
 
       FROM dim_asset
-      JOIN (select * from dim_site_asset WHERE site_id=#{options['site_id']}) dsa USING (asset_id) 
       JOIN fact_asset USING (asset_id)
       LEFT OUTER JOIN dim_operating_system on dim_asset.operating_system_id = dim_operating_system.operating_system_id
       LEFT OUTER JOIN dim_host_type USING (host_type_id)"
@@ -131,58 +130,133 @@ module NexposeServiceNow
     end
 
     def self.service_instance(options={})
-      "SELECT ds.Service_Name, asset_id as Nexpose_ID, port, dp.protocol, dsf.name
+      'SELECT ds.Service_Name, asset_id as Nexpose_ID, port, dp.protocol, dsf.name
               FROM fact_asset_scan_service
         LEFT OUTER JOIN (SELECT service_id, name as service_name FROM dim_service) ds USING (service_id)
               LEFT OUTER JOIN (SELECT service_fingerprint_id, name FROM dim_service_fingerprint) dsf USING (service_fingerprint_id)
         LEFT OUTER JOIN (SELECT protocol_id, name as protocol FROM dim_protocol) dp USING (protocol_id)
-        WHERE scan_id = lastScan(asset_id)"
+        WHERE scan_id = lastScan(asset_id)'
     end
 
 
-  #Need to wipe table each time
+    # Need to wipe table each time
     def self.group_accounts(options={})
-      "SELECT asset_id as Nexpose_ID, daga.name as Group_Account_Name
+      'SELECT asset_id as Nexpose_ID, daga.name as Group_Account_Name
         FROM dim_asset
-        JOIN dim_asset_group_account daga USING (asset_id)"
+        JOIN dim_asset_group_account daga USING (asset_id)'
     end
 
-  #Need to wipe table each time
+    # Need to wipe table each time
     def self.user_accounts(options={})
-      "SELECT da.asset_id as Nexpose_ID, daua.name as User_Account_Name,
+      'SELECT da.asset_id as Nexpose_ID, daua.name as User_Account_Name,
         daua.full_name as User_Account_Full_Name
 
         FROM dim_asset da
-        JOIN dim_asset_user_account daua USING (asset_id)"
+        JOIN dim_asset_user_account daua USING (asset_id)'
     end
 
-#Need to wipe table each time
+    # Need to wipe table each time
     def self.asset_groups(options={})
-      "SELECT asset_id as Nexpose_ID, dag.name as Asset_Group_Name,
+      'SELECT asset_id as Nexpose_ID, dag.name as Asset_Group_Name,
         dag.dynamic_membership, dag.description
         FROM dim_asset_group_asset daga
-        JOIN dim_asset_group dag on daga.asset_group_id = dag.asset_group_id"
+        JOIN dim_asset_group dag on daga.asset_group_id = dag.asset_group_id'
     end
 
-    #Need to wipe table each time
+    # Need to wipe table each time
     def self.sites(options={})
-      "SELECT asset_id as Nexpose_ID, ds.name as site_name
+      'SELECT asset_id as Nexpose_ID, ds.name as site_name
         FROM dim_asset
         JOIN dim_site_asset dsa USING (asset_id)
         JOIN dim_site ds on dsa.site_id = ds.site_id
-        ORDER BY ip_address"
+        ORDER BY ip_address'
     end
 
-    #Need to wipe table each time
+    # Need to wipe table each time
     def self.tags(options={})
-      "SELECT asset_id as Nexpose_ID, dt.tag_name
+      'SELECT asset_id as Nexpose_ID, dt.tag_name
         FROM dim_tag_asset dta
-        JOIN dim_tag dt on dta.tag_id = dt.tag_id"
+        JOIN dim_tag dt on dta.tag_id = dt.tag_id'
+    end
+
+    def self.generate_cve_filter(cves)
+      return '' if cves == nil || cves.empty?
+
+      cves = cves.map { |c| "reference='#{c}'" }.join(' OR ')
+
+      "JOIN (SELECT vulnerability_id, reference
+             FROM dim_vulnerability_reference
+             WHERE #{cves}) dvr USING (vulnerability_id)"
+    end
+
+    #TODO make sure that for max date first_discovered < date2
+    def self.generate_date_filter(dates, table_join=true)
+      return '' if dates == nil
+
+      # No filters applied, so no need to filter
+      return '' if dates.all? { |d| d == nil || d == '' }
+
+      min_date = dates.first
+      max_date = dates.last
+
+      # Version of vulnerable new items
+      unless table_join
+        filters = []
+        filters << "first_discovered >= '#{min_date}'" unless min_date.nil?
+        filters << "first_discovered <= '#{max_date}'" unless max_date.nil?
+        filters = filters.join(' AND ')
+        filters = "WHERE #{filters}"
+
+        return filters
+      end
+
+      date_filters = []
+
+      unless min_date.nil?
+        date_filters << "scan_finished > '#{min_date}'"
+      end
+      unless max_date.nil?
+        date_filters << "scan_started < '#{max_date}'"
+      end
+
+      condition = date_filters.join(' AND ')
+      "JOIN (SELECT scan_id, asset_id
+                           FROM fact_asset_scan
+                           WHERE #{condition}) fas
+                           ON fas.asset_id = da.asset_ID AND
+                           fas.scan_id = first_found"
+    end
+
+    def self.generate_cvss_filter(cvss_range)
+      return '' if cvss_range.nil? || cvss_range.last.nil?
+
+      cvss_min = cvss_range.first
+      cvss_max = cvss_range.last
+
+      # No need to join if not applying a filter
+      return '' if cvss_min.to_s == '0' && cvss_max.to_s == '10'
+
+      "JOIN (SELECT vulnerability_id, cvss_score
+             FROM dim_vulnerability
+             WHERE cvss_score >= #{cvss_min} AND cvss_score <= #{cvss_max}) dv
+             USING (vulnerability_id)"
     end
 
     def self.vulnerable_new_items(options={})
+      # self.send("vulnerable_new_items_per_#{options[:id_type]}", options)
+
+      standard_filter = if options[:id_type] == 'site'
+                          "MIN(fasv.scan_id) > #{options[:delta]}"
+                        else
+                          "MIN(fasv.date) > '#{options[:delta]}'"
+                        end
+
+      cve_filter = self.generate_cve_filter(options[:filters][:cve])
+      date_filter = self.generate_date_filter(options[:filters][:date], false)
+      cvss_filter = self.generate_cvss_filter(options[:filters][:cvss])
+
       "SELECT
-      coalesce(subq.host_name, CAST(subq.asset_id as text)) Configuration_Item,
+      CAST(subq.asset_id as text) Configuration_Item,
       TRUE as Active,
       concat('R7_', subq.vulnerability_id) as Vulnerability,
       fasva.first_discovered as First_Found,
@@ -193,17 +267,25 @@ module NexposeServiceNow
       coalesce(NULLIF(favi.name,''), 'None') as Protocol
 
       FROM (
-             SELECT fasv.asset_id, fasv.vulnerability_id, vulnerability_instances, 
-                     MIN(fasv.scan_id) as first_found, MAX(fasv.scan_id) as latest_found, 
+             SELECT fasv.asset_id, fasv.vulnerability_id, vulnerability_instances,
+                     MIN(fasv.scan_id) as first_found, MAX(fasv.scan_id) as latest_found,
                      s.current_scan, s.host_name, s.ip_address
               FROM fact_asset_scan_vulnerability_finding fasv
+
+              #{cve_filter}
+              #{cvss_filter}
+
               JOIN (
                   SELECT asset_id, host_name, ip_address, lastScan(asset_id) AS current_scan FROM dim_asset
                ) s ON s.asset_id = fasv.asset_id
                GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan, s.host_name, s.ip_address, vulnerability_instances
-               HAVING MIN(fasv.scan_id) > #{options[:last_scan_id]} AND MAX(fasv.scan_id)=current_scan
+               HAVING MAX(fasv.scan_id)=current_scan AND #{standard_filter}
               ) subq
-      JOIN (select asset_id, vulnerability_id, first_discovered, most_recently_discovered from fact_asset_vulnerability_age) fasva ON fasva.asset_id = subq.asset_id AND fasva.vulnerability_id = subq.vulnerability_id
+
+              JOIN (select asset_id, vulnerability_id,
+                           first_discovered, most_recently_discovered
+                    from fact_asset_vulnerability_age
+                    #{date_filter}) fasva USING (asset_id, vulnerability_id)
 
       JOIN (select DISTINCT on(asset_id, vulnerability_id) asset_id, scan_id, vulnerability_id, port, dp.name
                       from fact_asset_vulnerability_instance
@@ -212,23 +294,52 @@ module NexposeServiceNow
     end
 
     def self.vulnerable_old_items(options={})
+      standard_filter = if options[:id_type] == 'site'
+                          "MAX(fasv.scan_id) >= #{options[:delta]}"
+                        else
+                          "MAX(fasv.scan_id) >= scanAsOf(fasv.asset_id, '#{options[:delta]}') AND
+                           lastScan(fasv.asset_id) > scanAsOf(fasv.asset_id, '#{options[:delta]}')"
+                        end
+
+      cve_filter = self.generate_cve_filter(options[:filters][:cve])
+      date_filter = self.generate_date_filter(options[:filters][:date])
+      cvss_filter = self.generate_cvss_filter(options[:filters][:cvss])
+
+      # Only perform this operation is necessary
+      date_field = if date_filter.nil? || date_filter == ''
+                     ''
+                   else
+                     'MIN(fasv.scan_id) as first_found,'
+                   end
+
       "SELECT 
-      coalesce(da.host_name, CAST(da.asset_id as text)) Configuration_Item,
+      CAST(da.asset_id as text) Configuration_Item,
       FALSE as Active,
       concat('R7_', subq.vulnerability_id) as Vulnerability,
       da.ip_address as IP_Address
       FROM (
-             SELECT fasv.asset_id, fasv.vulnerability_id, MAX(fasv.scan_id) as latest_found, 
+             SELECT fasv.asset_id, fasv.vulnerability_id,
+                     #{date_field}
+                     MAX(fasv.scan_id) as latest_found,
                      s.current_scan, s.host_name, s.ip_address
               FROM fact_asset_scan_vulnerability_finding fasv
+
+              #{cve_filter}
+              #{cvss_filter}
+
               JOIN (
-                  SELECT asset_id, host_name, ip_address, lastScan(asset_id) AS current_scan FROM dim_asset
-               ) s ON s.asset_id = fasv.asset_id
-               GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan, s.host_name, s.ip_address, vulnerability_instances
-               HAVING MAX(fasv.scan_id) < current_scan AND MAX(fasv.scan_id) >= #{options[:last_scan_id]} 
-              ) subq
+                 SELECT asset_id, host_name, ip_address, lastScan(asset_id) AS current_scan FROM dim_asset
+              ) s ON s.asset_id = fasv.asset_id
+              GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan, s.host_name, s.ip_address, vulnerability_instances
+
+              HAVING MAX(fasv.scan_id) < current_scan
+                     AND #{standard_filter}
+             ) subq
+
       JOIN dim_asset da ON subq.asset_id = da.asset_id
+      #{date_filter}
       ORDER BY da.ip_address"
+
     end
 
     def self.latest_scans(options={})
@@ -238,8 +349,8 @@ module NexposeServiceNow
     end
 
     def self.multiple_reports?(query_name)
-      single_queries = ['vulnerabilities', 'vulnerability_category', 
-                        'vulnerability_references', 'latest_scans']
+      single_queries = %w(vulnerabilities vulnerability_category
+                          vulnerability_references latest_scans)
       return !(single_queries.include? query_name.to_s)
     end
   end