nexpose_ticketing 0.8.1 → 0.8.2

data/lib/nexpose_ticketing/ticket_service.rb

@@ -56,7 +56,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     TICKET_SERVICE_CONFIG_PATH =  File.join(File.dirname(__FILE__), '/config/ticket_service.config')
     LOGGER_FILE = File.join(File.dirname(__FILE__), '/log/ticket_service.log')
 
-    attr_accessor :helper_data, :nexpose_data, :options, :ticket_repository, :first_time, :nexpose_site_histories
+    attr_accessor :helper_data, :nexpose_data, :options, :ticket_repository, :first_time, :nexpose_item_histories
 
     def setup(helper_data)
       # Gets the Ticket Service configuration.
@@ -106,8 +106,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     end
 
     # Prepares all the local and nexpose historical data.
-    def prepare_historical_data(ticket_repository, options,
-        historical_scan_file = File.join(File.dirname(__FILE__), "#{options[:file_name]}"))
+    def prepare_historical_data(ticket_repository, options)
+      (options[:tag_run]) ?
+          historical_scan_file = File.join(File.dirname(__FILE__), "#{options[:tag_file_name]}") :
+          historical_scan_file = File.join(File.dirname(__FILE__), "#{options[:file_name]}")
+
       if File.exists?(historical_scan_file)
         log_message("Reading historical CSV file: #{historical_scan_file}.")
         file_site_histories = ticket_repository.read_last_scans(historical_scan_file)
@@ -119,32 +122,84 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     # Generates a full site(s) report ticket(s).
     def all_site_report(ticket_repository, options, helper)
-      log_message('Generating full vulnerability report on user entered sites.')
-      sites_to_query =  Array(options[:sites])
-
-      log_message("Generating full vulnerability report on the following sites: #{sites_to_query.join(', ')}")
-
-      sites_to_query.each { |site|
-        log_message("Running full vulnerability report on site #{site}")
-        all_vulns_file = ticket_repository.all_vulns(options, site)
+      if(options[:tag_run])
+        log_message('Generating full vulnerability report on user entered tags.')
+        items_to_query = Array(options[:tags])
+        log_message("Generating full vulnerability report on the following tags: #{items_to_query}")
+      else
+        log_message('Generating full vulnerability report on user entered sites.')
+        items_to_query =  Array(options[:sites])
+        log_message("Generating full vulnerability report on the following sites: #{items_to_query.join(', ')}")
+      end
+      items_to_query.each { |item|
+        log_message("Running full vulnerability report on item #{item}")
+        all_vulns_file = ticket_repository.all_vulns(options, item)
         log_message('Preparing tickets.')
-        ticket_rate_limiter(options, all_vulns_file, Proc.new { |ticket_batch| helper.prepare_create_tickets(ticket_batch, site) }, Proc.new { |tickets| helper.create_tickets(tickets) })
+        ticket_rate_limiter(options, all_vulns_file, Proc.new { |ticket_batch| helper.prepare_create_tickets(ticket_batch, options[:tag_run] ? "T#{item}" : item) }, Proc.new { |tickets| helper.create_tickets(tickets) })
       }
+
+      if(options[:tag_run])
+        items_to_query. each { |item_id|
+        tag_assets_historic_file = File.join(File.dirname(__FILE__), 'tag_assets', "#{options[:tag_file_name]}_#{item_id}.csv")
+        ticket_repository.generate_tag_asset_list(tags: item_id,
+                                csv_file: tag_assets_historic_file)
+        }
+      end
       log_message('Finished process all vulnerabilities.')
     end
 
     # There's possibly a new scan with new data.
     def delta_site_report(ticket_repository, options, helper, file_site_histories)
-      # Compares the Scan information from the File && Nexpose.
+      # Compares the scan information from file && Nexpose.
       no_processing = true
-      @nexpose_site_histories.each do |site_id, scan_id|
-        # There's no entry in the file, so it's either a new site in Nexpose or a new site we have to monitor.
-        if file_site_histories[site_id].nil? || file_site_histories[site_id] == -1
-          full_new_site_report(site_id, ticket_repository, options, helper)
+      @nexpose_item_histories.each do |item_id, last_scan_id|
+        # There's no entry in the file, so it's either a new item in Nexpose or a new item we have to monitor.
+        if file_site_histories[item_id].nil? || file_site_histories[item_id] == -1
+          full_new_site_report(item_id, ticket_repository, options, helper)
+          if(options[:tag_run])
+            tag_assets_historic_file = File.join(File.dirname(__FILE__), 'tag_assets', "#{options[:tag_file_name]}_#{item_id}.csv")
+            ticket_repository.generate_tag_asset_list(tags: item_id,
+                                                      csv_file: tag_assets_historic_file)
+          end
           no_processing = false
           # Site has been scanned since last seen according to the file.
-        elsif file_site_histories[site_id].to_i < nexpose_site_histories[site_id]
-          delta_site_new_scan(ticket_repository, site_id, options, helper, file_site_histories)
+        elsif file_site_histories[item_id].to_s != nexpose_item_histories[item_id].to_s
+          if(options[:tag_run])
+            # It's a tag run and something has changed (new/removed asset or new scan ID for an asset). To find out what, we must compare
+            # All tag assets and their scan IDs. Firstly we fetch all the assets in the tags
+            # in the configuration file and store them temporarily
+            tag_assets_tmp_file = File.join(File.dirname(__FILE__), "/tag_assets/#{options[:tag_file_name]}_#{item_id}.tmp")
+            tag_assets_historic_file = File.join(File.dirname(__FILE__), "/tag_assets/#{options[:tag_file_name]}_#{item_id}.csv")
+            ticket_repository.generate_tag_asset_list(tags: item_id,
+                                    csv_file: tag_assets_tmp_file)
+            new_tag_configuration = ticket_repository.read_tag_asset_list(tag_assets_tmp_file)
+            historic_tag_configuration = ticket_repository.read_tag_asset_list(tag_assets_historic_file)
+            #Compare the assets within the tags and their scan histories to find the ones we need to query
+            changed_assets = Hash[*(historic_tag_configuration.to_a - new_tag_configuration.to_a).flatten]
+            new_assets = Hash[*(new_tag_configuration.to_a - historic_tag_configuration.to_a).flatten]
+            new_assets.delete_if {|asset_id, scan_id| historic_tag_configuration.has_key?(asset_id.to_s)}
+            #all_assets_changed = new_assets.merge(changed_assets)
+            changed_assets.each do |asset_id, scan_id|
+              delta_site_new_scan(ticket_repository, asset_id, options, helper, changed_assets, item_id)
+            end
+            new_assets.each do |asset_id, scan_id|
+              #Since no previous scan IDs - we generate a full report.
+              options[:nexpose_item] = asset_id
+              full_new_site_report(item_id, ticket_repository, options, helper)
+              options.delete(:nexpose_item)
+            end
+          else
+            delta_site_new_scan(ticket_repository, item_id, options, helper, file_site_histories)
+          end
+          #Update the historic file
+          new_tag_asset_list = historic_tag_configuration.merge(new_tag_configuration)
+          trimmed_csv = []
+          trimmed_csv << 'asset_id, last_scan_id'
+          new_tag_asset_list.each do |asset_id, last_scan_id|
+            trimmed_csv << "#{asset_id},#{last_scan_id}"
+          end
+          ticket_repository.save_to_file(tag_assets_historic_file, trimmed_csv)
+          File.delete(tag_assets_tmp_file)
           no_processing = false
         end
       end
@@ -155,46 +210,48 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     end
 
     # There's a new site we haven't seen before.
-    def full_new_site_report(site_id, ticket_repository, options, helper)
-      log_message("New site id: #{site_id} detected. Generating report.")
-      new_site_vuln_file = ticket_repository.all_vulns(options, site_id)
+    def full_new_site_report(nexpose_item, ticket_repository, options, helper)
+      log_message("New nexpose id: #{nexpose_item} detected. Generating report.")
+      new_item_vuln_file = ticket_repository.all_vulns(options, nexpose_item)
       log_message('Report generated, preparing tickets.')
-      ticket_rate_limiter(options, new_site_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch, site_id)}, Proc.new {|tickets| helper.create_tickets(tickets)})
+      ticket_rate_limiter(options, new_item_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch, options[:tag_run] ? "T#{nexpose_item}" : nexpose_item)}, Proc.new {|tickets| helper.create_tickets(tickets)})
     end
 
     # There's a new scan with possibly new vulnerabilities.
-    def delta_site_new_scan(ticket_repository, site_id, options, helper, file_site_histories)
-      log_message("New scan detected for site: #{site_id}. Generating report.")
+    def delta_site_new_scan(ticket_repository, nexpose_item, options, helper, file_site_histories, tag_id=nil)
+      log_message("New scan detected for nexpose id: #{nexpose_item}. Generating report.")
       
       if options[:ticket_mode] == 'I' || options[:ticket_mode] == 'V'
         # I-mode and V-mode tickets require updating the tickets in the target system.
-        log_message("Scan id for new scan: #{file_site_histories[site_id]}.")
-        all_scan_vuln_file = ticket_repository.all_vulns_sites(scan_id: file_site_histories[site_id],
-                                                               site_id: site_id,
+        log_message("Scan id for new scan: #{file_site_histories[nexpose_item]}.")
+        all_scan_vuln_file = ticket_repository.all_vulns_since(scan_id: file_site_histories[nexpose_item],
+                                                               nexpose_item: nexpose_item,
                                                                severity: options[:severity],
                                                                ticket_mode: options[:ticket_mode],
                                                                riskScore: options[:riskScore],
                                                                vulnerabilityCategories: options[:vulnerabilityCategories],
-                                                               tags: options[:tags])
+                                                               tag_run: options[:tag_run],
+                                                               tag: tag_id)
 
         if helper.respond_to?('prepare_update_tickets') && helper.respond_to?('update_tickets')
-          ticket_rate_limiter(options, all_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_update_tickets(ticket_batch, site_id)}, Proc.new {|tickets| helper.update_tickets(tickets)})
+          ticket_rate_limiter(options, all_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_update_tickets(ticket_batch, tag_id.nil? ? nexpose_item : "T#{tag_id}")}, Proc.new {|tickets| helper.update_tickets(tickets)})
         else
           log_message('Helper does not implement update methods')
           fail "Helper using 'I' or 'V' mode must implement prepare_updates and update_tickets"
         end
 
         if options[:close_old_tickets_on_update] == 'Y'
-          tickets_to_close_file = ticket_repository.tickets_to_close(scan_id: file_site_histories[site_id],
-                                                                     site_id: site_id,
+          tickets_to_close_file = ticket_repository.tickets_to_close(scan_id: file_site_histories[nexpose_item],
+                                                                     nexpose_item: nexpose_item,
                                                                      severity: options[:severity],
                                                                      ticket_mode: options[:ticket_mode],
                                                                      riskScore: options[:riskScore],
                                                                      vulnerabilityCategories: options[:vulnerabilityCategories],
-                                                                     tags: options[:tags])
+                                                                     tag_run: options[:tag_run],
+                                                                     tag: tag_id)
 
           if helper.respond_to?('prepare_close_tickets') && helper.respond_to?('close_tickets')
-            ticket_rate_limiter(options, tickets_to_close_file, Proc.new {|ticket_batch| helper.prepare_close_tickets(ticket_batch, site_id)}, Proc.new {|tickets| helper.close_tickets(tickets)})
+            ticket_rate_limiter(options, tickets_to_close_file, Proc.new {|ticket_batch| helper.prepare_close_tickets(ticket_batch, tag_id.nil? ? nexpose_item : "T#{tag_id}")}, Proc.new {|tickets| helper.close_tickets(tickets)})
           else
             log_message('Helper does not implement close methods')
             fail 'Helper using \'I\' or \'V\' mode must implement prepare_close_tickets and close_tickets'
@@ -202,36 +259,38 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         end
       else
         # D-mode tickets require creating new tickets and closing old tickets.
-        new_scan_vuln_file = ticket_repository.new_vulns_sites(scan_id: file_site_histories[site_id],
-                                                               site_id: site_id,
+        new_scan_vuln_file = ticket_repository.new_vulns(scan_id: file_site_histories[nexpose_item],
+                                                               nexpose_item: nexpose_item,
                                                                severity: options[:severity],
                                                                ticket_mode: options[:ticket_mode],
                                                                riskScore: options[:riskScore],
                                                                vulnerabilityCategories: options[:vulnerabilityCategories],
-                                                               tags: options[:tags])
+                                                               tag_run: options[:tag_run],
+                                                               tag: tag_id)
 
         preparse = CSV.new(new_scan_vuln_file.path, headers: :first_row)
         empty_report = preparse.shift.nil?
-        log_message("No new vulnerabilities found in new scan for site: #{site_id}.") if empty_report
-        log_message("New vulnerabilities found in new scan for site #{site_id}, preparing tickets.") unless empty_report
+        log_message("No new vulnerabilities found in new scan for site: #{nexpose_item}.") if empty_report
+        log_message("New vulnerabilities found in new scan for site #{nexpose_item}, preparing tickets.") unless empty_report
         unless empty_report
-          ticket_rate_limiter(options, new_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch, site_id)}, Proc.new {|tickets| helper.create_tickets(tickets)})
+          ticket_rate_limiter(options, new_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch, tag_id.nil? ? nexpose_item : "T#{tag_id}")}, Proc.new {|tickets| helper.create_tickets(tickets)})
         end
         
         if helper.respond_to?('prepare_close_tickets') && helper.respond_to?('close_tickets')
-          old_scan_vuln_file = ticket_repository.old_vulns_sites(scan_id: file_site_histories[site_id],
-                                                                 site_id: site_id,
+          old_scan_vuln_file = ticket_repository.old_vulns(scan_id: file_site_histories[nexpose_item],
+                                                                 site_id: nexpose_item,
                                                                  severity: options[:severity],
                                                                  riskScore: options[:riskScore],
                                                                  vulnerabilityCategories: options[:vulnerabilityCategories],
-                                                                 tags: options[:tags])
+                                                                 tag_run: options[:tag_run],
+                                                                 tag: tag_id)
 
           preparse = CSV.new(old_scan_vuln_file.path, headers: :first_row, :skip_blanks => true)
           empty_report = preparse.shift.nil?
-          log_message("No old (closed) vulnerabilities found in new scan for site: #{site_id}.") if empty_report
-          log_message("Old vulnerabilities found in new scan for site #{site_id}, preparing closures.") unless empty_report
+          log_message("No old (closed) vulnerabilities found in new scan for site: #{nexpose_item}.") if empty_report
+          log_message("Old vulnerabilities found in new scan for site #{nexpose_item}, preparing closures.") unless empty_report
           unless empty_report
-            ticket_rate_limiter(options, old_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_close_tickets(ticket_batch, site_id)}, Proc.new {|tickets| helper.close_tickets(tickets)})
+            ticket_rate_limiter(options, old_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_close_tickets(ticket_batch, tag_id.nil? ? nexpose_item : "T#{tag_id}")}, Proc.new {|tickets| helper.close_tickets(tickets)})
           end
         else
           # Create a log message but do not halt execution of the helper if ticket closing is not
@@ -315,42 +374,50 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     # Starts the Ticketing Service.
     def start
+      #Decide if this is a tag run (tags always override sites as the API does not allow for the combination of the two)
+      @options[:tag_run] = !@options[:tags].nil? && !@options[:tags].empty?
+
       # Checks if the csv historical file already exists && reads it, otherwise create it && assume first time run.
       file_site_histories = prepare_historical_data(@ticket_repository, @options)
       historical_scan_file = File.join(File.dirname(__FILE__), "#{@options[:file_name]}")
+      historical_tag_file = File.join(File.dirname(__FILE__), "#{@options[:tag_file_name]}")
+
       # If we didn't specify a site || first time run (no scan history), then it gets all the vulnerabilities.
-      if @options[:sites].nil? || @options[:sites].empty? || file_site_histories.nil?
-        log_message('Storing current scan state before obtaining all vulnerabilities.')
+      if (((@options[:sites].nil? || @options[:sites].empty? || file_site_histories.nil?) && !@options[:tag_run]) || (@options[:tag_run] && file_site_histories.nil?))
+      log_message('Storing current scan state before obtaining all vulnerabilities.')
+        current_scan_state = ticket_repository.load_last_scans(@options)
 
-        if options[:sites].nil? || options[:sites].empty?
+        if (options[:sites].nil? || options[:sites].empty?) && (!@options[:tag_run])
           log_message('No site(s) specified, generating for all sites.')
           @ticket_repository.all_site_details.each { |site|  (@options[:sites] ||= []) << site.id.to_s }
           log_message("List of sites is now <#{@options[:sites]}>")
         end
 
-        current_scan_state = ticket_repository.load_last_scans(@options)
-
         all_site_report(@ticket_repository, @options, @helper)
 
         #Generate historical CSV file after completing the fist query.
         log_message('No historical CSV file found. Generating.')
-        @ticket_repository.save_scans_to_file(historical_scan_file, current_scan_state)
+        @options[:tag_run] ?
+            @ticket_repository.save_to_file(historical_tag_file, current_scan_state) :
+            @ticket_repository.save_to_file(historical_scan_file, current_scan_state)
         log_message('Historical CSV file generated.')
       else
         log_message('Obtaining last scan information.')
-        @nexpose_site_histories = @ticket_repository.last_scans(@options)
+        @nexpose_item_histories = @ticket_repository.last_scans(@options)
 
         # Scan states can change during our processing. Store the state we are
-        # about to process and move this to the historical_scan_file if we
+        # about to process and move this to the historical file if we
         # successfully process.
         log_message('Calculated deltas, storing current scan state.')
         current_scan_state = ticket_repository.load_last_scans(options)
 
         # Only run if a scan has been ran ever in Nexpose.
-        unless @nexpose_site_histories.empty?
+        unless @nexpose_item_histories.empty?
           delta_site_report(@ticket_repository, @options, @helper, file_site_histories)
           # Processing completed successfully. Update historical scan file.
-          @ticket_repository.save_scans_to_file(historical_scan_file, current_scan_state)
+          @options[:tag_run] ?
+              @ticket_repository.save_to_file(historical_tag_file, current_scan_state) :
+              @ticket_repository.save_to_file(historical_scan_file, current_scan_state)
         end
       end
       log_message('Exiting ticket service.')

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: nexpose_ticketing
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.2
 platform: ruby
 authors:
 - Damian Finol
+- JJ Cassidy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-22 00:00:00.000000000 Z
+date: 2015-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nexpose
@@ -95,7 +96,7 @@ dependencies:
 description: This gem provides a Ruby implementation of different integrations with
   ticketing services for Nexpose.
 email:
-- damian_finol@rapid7.com
+- integrations_support@rapid7.com
 executables:
 - nexpose_jira
 - nexpose_servicenow