nexpose_ticketing 0.2.3 → 0.3.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NjMwMjMyNTVlNjIyMmY2YTNjMzc5ZTVmODFhZDdiZjJjNDI2YzE2OA==
-  data.tar.gz: !binary |-
-    YTczODY0MjMzYTg3OTQwMTA1ZDZmMjk1ODdhZmJiZGQ2MGUxOTRjNA==
+SHA1:
+  metadata.gz: 8b5571a811b41e5db9bad42f938687414f521da3
+  data.tar.gz: 97c1ea6f3932f5b37138f15128f2530eb6bae9f1
 SHA512:
-  metadata.gz: !binary |-
-    ZmUzZDhmMDMxYjBiY2RiOTkyMWNhNDQwMjMwNmM1ZDBlMWM1ZWQxMmVjNDY4
-    MDRlOWZhYTgyZGY2ZWJmZjhlYmYyYWMzMGE4ZmVjOTZlNjNiYmY5NTEwNmI4
-    NGQ5OWZhMzExYTAyYjg0Y2M5ZjViNmYwYzg2ZGI2NGIxZGRjNzc=
-  data.tar.gz: !binary |-
-    ZmFiNWE4MjAzMzk1N2ExMTFiMWNmZTM4NjIxNjk1MzE2ZjE2OTVmOGVlNjU3
-    ZGM5MmY0YzZkMmZkZjViMWIwMjFhZTJhM2M3NWIzZmUwNTNiNzk0MjJiMWQy
-    OWVkZjY2ZmYzNTkxMGJlOWNkMGU4ZjUxNjkwYTYyYTI2N2M0N2Y=
+  metadata.gz: a34d4c2ae876184066c674ba39dfa7c2fa792b476c10722d2da0dcd2e4ef8516062f5dfcd8e317a80fc4abfd8001b4b3b8f32cd61d6cf10a1012e561919dd217
+  data.tar.gz: a039db4db30f7fdf24a9f56c3851533358085226da163f8a8857abbabce71637e38029c39f48e334926a931abd3c4a0c099058b7e173edc67e94c01d0e181c5c

data/bin/nexpose_servicenow

@@ -1,4 +1,4 @@
-#!/usr/bin/env ruby
+#!/usr/bin/ruby
 require 'yaml'
 require 'nexpose_ticketing'
 

data/lib/nexpose_ticketing/config/servicenow.config

@@ -7,11 +7,11 @@
 :helper_name: ServiceNowHelper
 
 # (M) ServiceNow url (currently requires using JSON)
-:servicenow_url: https://myinstance.service-now.com/incident.do?JSON
+:servicenow_url: https://your.service-now.com/incident.do?JSON
 # (M) Username for ServiceNow
 :username: admin
 # (M) Password for above username
-:password: password
+:password: admin
 # (M) If 'Y', SSL connections will output to stderr (default is 'N')
 :verbose_mode: N
 # (M) Amount of times the helper will follow 301/302 redirections

data/lib/nexpose_ticketing/config/ticket_service.config

@@ -8,7 +8,7 @@
   :logging_enabled: true
   # Filters the reports to specific sites one per line, leave empty for no site.
   :sites:
-  - '1'
+  - '7'
   # Minimum floor severity to report on. Number between 0 and 10.
   :severity: 8
   # (M) Name of the report historial file saved in disk.
@@ -17,10 +17,14 @@
   #   'D' Default IP *-* Vulnerability
   #   'I' IP address -* Vulnerability
   :ticket_mode: I
+  # Timeout in seconds. The number of seconds the GEM waits for a response from Nexpose before exiting.
+  :timeout: 10800
+  # Ticket batching. Breaks ticket processing into groups of value size controlling resource utilisation of both systems.
+  :batch_size: 300
 # Nexpose options.
 :nexpose_data:
   # (M) Nexpose console hostname.
-  :nxconsole: 127.0.0.1
+  :nxconsole: nexpose-rh.dev.lax.rapid7.com
   # (M) Nexpose username.
   :nxuser: nxadmin
   # (M) Nexpose password.

data/lib/nexpose_ticketing/helpers/servicenow_helper.rb

@@ -176,7 +176,7 @@ class ServiceNowHelper
     tickets = []
     current_ip = -1
     CSV.parse(vulnerability_list.chomp, headers: :first_row)  do |row|
-      if current_ip == -1 
+      if current_ip == -1
         current_ip = row['ip_address']
         @log.log_message("Creating ticket with IP address: #{row['ip_address']}")
         @ticket = {
@@ -187,24 +187,25 @@ class ServiceNowHelper
           'urgency' => '1',
           'short_description' => "#{row['ip_address']} => Vulnerabilities",
           'work_notes' => "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++
-                           ++ New Vulnerabilities +++++++++++++++++++++++++++++++++++++
+                           ++ New Vulnerabilities ++++++++++++++++++++++++++++++++++++
                            +++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
         }
       end
       if current_ip == row['ip_address']
         @ticket['work_notes'] += 
-          "==========================================
+          "\n\n==========================================
           Summary: #{row['summary']}
           ----------------------------------------------------------------------------
           Fix: #{row['fix']}"
         unless row['url'].nil?
           @ticket['work_notes'] += 
-            "----------------------------------------------------------------------------
+            "\n----------------------------------------------------------------------------
              URL: #{row['url']}"
         end
       end
       unless current_ip == row['ip_address']
         # NXID in the work_notes is the unique identifier used to query incidents to update them.
+        @log.log_message("Found new IP address. Finishing ticket with with IP address: #{current_ip} and moving onto IP #{row['ip_address']}")
         @ticket['work_notes'] += "\nNXID: #{current_ip}"
         @ticket = @ticket.to_json
         tickets.push(@ticket)
@@ -213,7 +214,7 @@ class ServiceNowHelper
       end
     end
     # NXID in the work_notes is the unique identifier used to query incidents to update them.
-    @ticket['work_notes'] += "\nNXID: #{current_ip}"
+    @ticket['work_notes'] += "\nNXID: #{current_ip}" unless (@ticket.size == 0)
     tickets.push(@ticket.to_json) unless @ticket.nil?
     tickets
   end
@@ -242,8 +243,12 @@ class ServiceNowHelper
         ticket_status = row['comparison']
         @log.log_message("Creating ticket update with IP address: #{row['ip_address']}")
         @log.log_message("Ticket status #{ticket_status}")
+        action =  'update'
+        if ticket_status == 'New'
+          action =  'insert'
+        end
         @ticket = {
-          'sysparm_action' => 'update',
+          'sysparm_action' => action,
           'sysparm_query' => "work_notesCONTAINSNXID: #{row['ip_address']}",
           'work_notes' => 
             "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -283,7 +288,7 @@ class ServiceNowHelper
       end
     end
     # NXID in the work_notes is the unique identifier used to query incidents to update them.
-    @ticket['work_notes'] += "\nNXID: #{current_ip}"
+    @ticket['work_notes'] += "\nNXID: #{current_ip}" unless (@ticket.size == 0)
     tickets.push(@ticket.to_json) unless @ticket.nil?
     tickets
   end

data/lib/nexpose_ticketing/queries.rb

@@ -19,7 +19,7 @@ module NexposeTicketing
     #    |url| |summary| |fix|
     #
     def self.all_new_vulns
-      "SELECT subs.asset_id, da.ip_address, subs.current_scan, subs.vulnerability_id, davs.solution_id, ds.nexpose_id,
+	"SELECT DISTINCT on (da.ip_address, davs.solution_id) subs.asset_id, da.ip_address, subs.current_scan, subs.vulnerability_id, davs.solution_id, ds.nexpose_id,
        ds.url,proofAsText(ds.summary) as summary, proofAsText(ds.fix) as fix
         FROM (SELECT fasv.asset_id, fasv.vulnerability_id, s.current_scan
           FROM fact_asset_scan_vulnerability_finding fasv
@@ -28,14 +28,14 @@ module NexposeTicketing
             SELECT asset_id, previousScan(asset_id) AS baseline_scan, lastScan(asset_id) AS current_scan
               FROM dim_asset) s
               ON s.asset_id = fasv.asset_id AND (fasv.scan_id = s.baseline_scan OR fasv.scan_id = s.current_scan)
-              GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan
-              HAVING baselineComparison(fasv.scan_id, current_scan) = 'New'
+              GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan, fasv.scan_id
+              HAVING NOT baselineComparison(fasv.scan_id, current_scan) = 'Old'
           ) subs
         JOIN dim_asset_vulnerability_solution davs USING (vulnerability_id)
         JOIN dim_solution ds USING (solution_id)
         JOIN dim_asset da ON subs.asset_id = da.asset_id
-        ORDER BY da.ip_address"
-    end
+        ORDER BY da.ip_address, davs.solution_id"
+	end
     
     # Gets all new vulnerabilities happening after a reported scan id.
     #

data/lib/nexpose_ticketing/report_helper.rb

@@ -0,0 +1,76 @@
+require 'nexpose'
+require 'securerandom'
+include Nexpose
+
+module NexposeReportHelper
+  class ReportOps
+
+    def initialize(nsc, timeout)
+      @timeout = timeout
+      @nsc = nsc
+    end
+
+    def generate_sql_report_config
+      random_name = "Nexpose-ticketing-Temp-#{SecureRandom.uuid}"
+      Nexpose::ReportConfig.new(random_name, nil, 'sql')
+    end
+
+    def save_generate_cleanup_report_config(report_config)
+      report_id = report_config.save(@nsc, false)
+      @nsc.generate_report(report_id, true)
+      wait_for_report(report_id)
+      report_details = @nsc.last_report(report_id)
+      file = Tempfile.new("#{report_id}")
+      file.write(@nsc.download(report_details.uri))
+      #Got the report, cleanup server-side
+      @nsc.delete_report_config(report_id)
+      file
+    end
+
+    # Wait for report generation to complete.
+    #
+    # @param [Fixnum] id Report configuration ID of the report waiting to generate.
+    #
+    def wait_for_report(id)
+      wait_until(:fail_on_exceptions => TRUE, :on_timeout => "Report generation timed out. Status: #{r = @nsc.last_report(id); r ? r.status : 'unknown'}") {
+        if %w(Failed Aborted Unknown).include?(@nsc.last_report(id).status)
+          raise "Report failed to generate! Status <#{@nsc.last_report(id).status}>"
+        end
+        @nsc.last_report(id).status == 'Generated'
+      }
+    end
+
+    # Wait for a given block to evaluate to true.
+    #
+    # The following flags are accepted as arguments:
+    #   :timeout Number of seconds to wait before timing out. Defaults to 90.
+    #   :polling_interval Number of seconds to wait between checking block again.
+    #      Defaults to 5.
+    #   :fail_on_exceptions Whether to fail fast. Defaults to off.
+    #   :on_timeout Message to raise with the exception when a timeout occurs.
+    #
+    # Example usage:
+    #   wait_until { site.id > 0 }
+    #   wait_until(:timeout => 30, :polling_interval => 0.5) { 1 == 2 }
+    #   wait_until(:on_timeout => 'Unable to confirm scan integration.') { console.sites.find { |site| site[:site_id] == site_id.to_i }[:risk_score] > 0.0 }
+    #
+    def wait_until(options = {})
+      polling_interval = 90
+      time_limit = Time.now + @timeout
+      loop do
+        begin
+          val = yield
+          return val if val
+        rescue Exception => error
+          raise error if options[:fail_on_exceptions]
+        end
+        if Time.now >= time_limit
+          raise options[:on_timeout] if options[:on_timeout]
+          error ||= 'Timed out waiting for condition.'
+          raise error
+        end
+        sleep polling_interval
+      end
+    end
+    end
+end

data/lib/nexpose_ticketing/ticket_repository.rb

@@ -4,10 +4,28 @@ module NexposeTicketing
     require 'csv'
     require 'nexpose'
     require 'nexpose_ticketing/queries'
+    require 'nexpose_ticketing/report_helper'
+
+    @timeout = 10800
+
+    def initialize(options = nil)
+      @timeout = options[:timeout]
+    end
 
     def nexpose_login(nexpose_data)
       @nsc = Nexpose::Connection.new(nexpose_data[:nxconsole], nexpose_data[:nxuser], nexpose_data[:nxpasswd])
       @nsc.login
+      #After login, create the report helper
+      @report_helper = NexposeReportHelper::ReportOps.new(@nsc, @timeout)
+    end
+
+    # Returns an array of all sites in the users environment.
+    #
+    # * *Returns* :
+    #   - An array of Nexpose::SiteSummary objects.
+    #
+    def all_site_details()
+      @nsc.sites
     end
 
     # Reads the site scan history from disk.
@@ -31,28 +49,56 @@ module NexposeTicketing
     # * *Args*    :
     #   - +csv_file_name+ -  CSV File name.
     #
-    def save_last_scans(csv_file_name, saved_file = nil, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
+    def save_last_scans(csv_file_name, options = {}, saved_file = nil)
+      current_scan_state = load_last_scans(options)
+      save_scans_to_file(csv_file_name, current_scan_state, saved_file)
+    end
+
+    # Loads the last scan info to memory.
+    #
+    # * *Args*    :
+    #   - +csv_file_name+ -  CSV File name.
+    #
+    def load_last_scans(options = {}, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
+      sites = Array(options[:sites])
       report_config.add_filter('version', '1.2.0')
       report_config.add_filter('query', Queries.last_scans)
-      report_output = report_config.generate(@nsc)
+
+      report_output = report_config.generate(@nsc, @timeout)
       csv_output = CSV.parse(report_output.chomp,  headers: :first_row)
-      saved_file.open(csv_file_name, 'w') { |file| file.puts(csv_output) } unless saved_file.nil?
+
+      #We only care about sites we are monitoring.
+      trimmed_csv = []
+      trimmed_csv << report_output.lines.first
+      csv_output.each  do |row|
+        if sites.include? row.to_s[0]
+          trimmed_csv << row
+        end
+      end
+      trimmed_csv
+    end
+
+    # Saves CSV scan information to disk
+    #
+    # * *Args*    :
+    #   - +csv_file_name+ -  CSV File name.
+    #
+    def save_scans_to_file(csv_file_name, trimmed_csv, saved_file = nil)
+      saved_file.open(csv_file_name, 'w') { |file| file.puts(trimmed_csv) } unless saved_file.nil?
       if saved_file.nil?
-        File.open(csv_file_name, 'w') { |file| file.puts(csv_output) }
+        File.open(csv_file_name, 'w') { |file| file.puts(trimmed_csv) }
       end
     end
 
-    # Gets the last scan information from nexpose.
+    # Gets the last scan information from nexpose sans the CSV headers.
     #
     # * *Returns* :
     #   - A hash with site_ids => last_scan_id
     #
-    def last_scans(report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
-      report_config.add_filter('version', '1.2.0')
-      report_config.add_filter('query', Queries.last_scans)
-      report_output = report_config.generate(@nsc).chomp
+    def last_scans(options = {})
       nexpose_sites = Hash.new(-1)
-      CSV.parse(report_output, headers: :first_row)  do |row|
+      trimmed_csv = load_last_scans(options)
+      trimmed_csv.drop(1).each  do |row|
         nexpose_sites[row['site_id']] = row['last_scan_id'].to_i
       end
       nexpose_sites
@@ -62,23 +108,29 @@ module NexposeTicketing
     #
     # * *Args*    :
     #   - +site_options+ -  A Hash with site(s) and severity level.
+    #   - +site_to_query+ - Override for user-configured site options
     #
     # * *Returns* :
     #   - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
     #     |url| |summary| |fix|
     #
-    def all_vulns(site_options = {}, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
-      sites = Array(site_options[:sites])
-      severity = site_options[:severity].nil? ? 0 : site_options[:severity]
+    def all_vulns(options = {}, site_override = nil)
+      if site_override.nil?
+        sites = Array(options[:sites])
+      else
+        sites = site_override
+      end
+      report_config =  @report_helper.generate_sql_report_config()
+      severity = options[:severity].nil? ? 0 : options[:severity]
       report_config.add_filter('version', '1.2.0')
       report_config.add_filter('query', Queries.all_new_vulns)
       unless sites.empty?
-        sites.each do |site_id|
+        Array(sites).each do |site_id|
           report_config.add_filter('site', site_id)
         end
       end
       report_config.add_filter('vuln-severity', severity)
-      report_config.generate(@nsc)
+      @report_helper.save_generate_cleanup_report_config(report_config)
     end
 
     # Gets the new vulns from base scan reported_scan_id and the newest / latest scan from a site.
@@ -90,7 +142,8 @@ module NexposeTicketing
     #   - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
     #     |url| |summary| |fix|
     #
-    def new_vulns_sites(site_options = {}, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
+    def new_vulns_sites(site_options = {})
+      report_config =  @report_helper.generate_sql_report_config()
       site = site_options[:site_id]
       reported_scan_id = site_options[:scan_id]
       fail 'Site cannot be null or empty' if site.nil? || reported_scan_id.nil?
@@ -99,7 +152,7 @@ module NexposeTicketing
       report_config.add_filter('query', Queries.new_vulns_since_scan(reported_scan_id))
       report_config.add_filter('site', site)
       report_config.add_filter('vuln-severity', severity)
-      report_config.generate(@nsc)
+      @report_helper.save_generate_cleanup_report_config(report_config)
     end
     
     # Gets the old vulns from base scan reported_scan_id and the newest / latest scan from a site.
@@ -111,7 +164,8 @@ module NexposeTicketing
     #   - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
     #     |url| |summary| |fix|
     #
-    def old_vulns_sites(site_options = {}, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
+    def old_vulns_sites(site_options = {})
+      report_config =  @report_helper.generate_sql_report_config()
       site = site_options[:site_id]
       reported_scan_id = site_options[:scan_id]
       fail 'Site cannot be null or empty' if site.nil? || reported_scan_id.nil?
@@ -120,7 +174,7 @@ module NexposeTicketing
       report_config.add_filter('query', Queries.old_vulns_since_scan(reported_scan_id))
       report_config.add_filter('site', site)
       report_config.add_filter('vuln-severity', severity)
-      report_config.generate(@nsc)
+      @report_helper.save_generate_cleanup_report_config(report_config)
     end
     
     # Gets all vulns from base scan reported_scan_id and the newest / latest scan from a site. This is
@@ -133,7 +187,8 @@ module NexposeTicketing
     #   - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
     #     |url| |summary| |fix| |comparison| 
     #
-    def all_vulns_sites(site_options = {}, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
+    def all_vulns_sites(site_options = {})
+      report_config =  @report_helper.generate_sql_report_config()
       site = site_options[:site_id]
       reported_scan_id = site_options[:scan_id]
       fail 'Site cannot be null or empty' if site.nil? || reported_scan_id.nil?
@@ -142,7 +197,7 @@ module NexposeTicketing
       report_config.add_filter('query', Queries.all_vulns_since_scan(reported_scan_id))
       report_config.add_filter('site', site)
       report_config.add_filter('vuln-severity', severity)
-      report_config.generate(@nsc)
+      @report_helper.save_generate_cleanup_report_config(report_config)
     end
   end
 end

data/lib/nexpose_ticketing/ticket_service.rb

@@ -81,9 +81,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
       end
       log_message("Enabling helper: #{@helper_data[:helper_name]}.")
       @helper = eval(@helper_data[:helper_name]).new(@helper_data, @options)
-      @ticket_repository = NexposeTicketing::TicketRepository.new
+
+      log_message("Creating ticketing repository with timeout value: #{@options[:timeout]}.")
+      @ticket_repository = NexposeTicketing::TicketRepository.new(options)
       @ticket_repository.nexpose_login(@nexpose_data)
-      @first_time = false
     end
 
     def setup_logging(enabled = false)
@@ -109,36 +110,40 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         log_message("Reading historical CSV file: #{historical_scan_file}.")
         file_site_histories = ticket_repository.read_last_scans(historical_scan_file)
       else
-        log_message('No historical CSV file found. Generating.')
-        ticket_repository.save_last_scans(historical_scan_file)
-        log_message('Historical CSV file generated.')
-        file_site_histories = ticket_repository.read_last_scans(historical_scan_file)
-        @first_time = true
+        file_site_histories = nil
       end
       file_site_histories
     end
 
     # Generates a full site(s) report ticket(s).
-    def all_site_report(ticket_repository, options, helper,
-        historical_scan_file = File.join(File.dirname(__FILE__), "#{options[:file_name]}"))
-      log_message('First time run, generating full vulnerability report.') if @first_time
-      log_message('No site(s) specified, generating full vulnerability report.') if options[:sites].empty?
-      all_delta_vulns = ticket_repository.all_vulns(severity: options[:severity])
-      log_message('Preparing tickets.')
-      tickets = helper.prepare_create_tickets(all_delta_vulns)
-      helper.create_tickets(tickets)
-      log_message("Done processing, updating historical CSV file #{historical_scan_file}.")
-      ticket_repository.save_last_scans(historical_scan_file)
-      log_message('Done updating historical CSV file, service shutting down.')
+    def all_site_report(ticket_repository, options, helper)
+
+      sites_to_query = Array.new
+      if options[:sites].empty?
+        log_message('No site(s) specified, generating full vulnerability report.')
+        @ticket_repository.all_site_details.each { |site| sites_to_query << site.id }
+      else
+        log_message('Generating full vulnerability report on user entered sites.')
+        sites_to_query =  Array(options[:sites])
+      end
+
+      log_message("Generating full vulnerability report on the following sites: #{sites_to_query.join(', ')}")
+
+      sites_to_query.each { |site|
+        log_message("Running full vulnerability report on site #{site}")
+        all_vulns_file = ticket_repository.all_vulns(options, site)
+        log_message('Preparing tickets.')
+        ticket_rate_limiter(options, all_vulns_file, Proc.new { |ticket_batch| helper.prepare_create_tickets(ticket_batch) }, Proc.new { |tickets| helper.create_tickets(tickets) })
+      }
+      log_message('Finished process all vulnerabilities.')
     end
 
     # There's possibly a new scan with new data.
-    def delta_site_report(ticket_repository, options, helper, file_site_histories,
-          historical_scan_file = File.join(File.dirname(__FILE__), "#{options[:file_name]}"))
+    def delta_site_report(ticket_repository, options, helper, file_site_histories)
       # Compares the Scan information from the File && Nexpose.
       no_processing = true
       @nexpose_site_histories.each do |site_id, scan_id|
-        # There's no entry in the file, so it's a new site in Nexpose.
+        # There's no entry in the file, so it's either a new site in Nexpose or a new site we have to monitor.
         if file_site_histories[site_id].nil? || file_site_histories[site_id] == -1
           full_new_site_report(site_id, ticket_repository, options, helper)
           no_processing = false
@@ -151,18 +156,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
       # Done processing, update the CSV to the latest scan info.
       log_message("Nothing new to process, updating historical CSV file #{options[:file_name]}.") if no_processing
       log_message("Done processing, updating historical CSV file #{options[:file_name]}.") unless no_processing
-      ticket_repository.save_last_scans(historical_scan_file)
-      log_message('Done updating historical CSV file, service shutting down.')
       no_processing
     end
 
     # There's a new site we haven't seen before.
     def full_new_site_report(site_id, ticket_repository, options, helper)
       log_message("New site id: #{site_id} detected. Generating report.")
-      new_site_vuln = ticket_repository.all_vulns(sites: [site_id], severity: options[:severity])
+      new_site_vuln_file = ticket_repository.all_vulns(sites: [site_id], severity: options[:severity])
       log_message('Report generated, preparing tickets.')
-      ticket = helper.prepare_create_tickets(new_site_vuln)
-      helper.create_tickets(ticket)
+      ticket_rate_limiter(options, new_site_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch)}, Proc.new {|tickets| helper.create_tickets(tickets)})
     end
 
     # There's a new scan with possibly new vulnerabilities.
@@ -172,63 +174,143 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
       if options[:ticket_mode] == 'I'
         # I-mode tickets require updating the tickets in the target system.
         log_message("Scan id for new scan: #{file_site_histories[site_id]}.")
-        all_scan_vuln = ticket_repository.all_vulns_sites(scan_id: file_site_histories[site_id], 
+        all_scan_vuln_file = ticket_repository.all_vulns_sites(scan_id: file_site_histories[site_id],
                                                           site_id: site_id,
                                                           severity: options[:severity])
         if helper.respond_to?("prepare_update_tickets") && helper.respond_to?("update_tickets")
-          tickets = helper.prepare_update_tickets(all_scan_vuln)
-          helper.update_tickets(tickets)
+          ticket_rate_limiter(options, all_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_update_tickets(ticket_batch)}, Proc.new {|tickets| helper.update_tickets(tickets)})
         else
           log_message("Helper does not implement update methods")
           fail "Helper using 'I' mode must implement prepare_updates and update_tickets"
         end
       else
         # D-mode tickets require creating new tickets and closing old tickets.
-        new_scan_vuln = ticket_repository.new_vulns_sites(scan_id: file_site_histories[site_id], site_id: site_id,
+        new_scan_vuln_file = ticket_repository.new_vulns_sites(scan_id: file_site_histories[site_id], site_id: site_id,
                                                           severity: options[:severity])
-        preparse = CSV.new(new_scan_vuln.chomp, headers: :first_row)
+        preparse = CSV.new(new_scan_vuln_file.path, headers: :first_row)
         empty_report = preparse.shift.nil?
         log_message("No new vulnerabilities found in new scan for site: #{site_id}.") if empty_report
         log_message("New vulnerabilities found in new scan for site #{site_id}, preparing tickets.") unless empty_report
         unless empty_report
-          tickets = helper.prepare_create_tickets(new_scan_vuln)
-          helper.create_tickets(tickets)
+          ticket_rate_limiter(options, new_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch)}, Proc.new {|tickets| helper.create_tickets(tickets)})
         end
         
         if helper.respond_to?("prepare_close_tickets") && helper.respond_to?("close_tickets")
-          old_scan_vuln = ticket_repository.old_vulns_sites(scan_id: file_site_histories[site_id], site_id: site_id,
+          old_scan_vuln_file = ticket_repository.old_vulns_sites(scan_id: file_site_histories[site_id], site_id: site_id,
                                                             severity: options[:severity])
-          preparse = CSV.new(old_scan_vuln.chomp, headers: :first_row)
+          preparse = CSV.new(old_scan_vuln_file.path, headers: :first_row, :skip_blanks => true)
           empty_report = preparse.shift.nil?
           log_message("No old (closed) vulnerabilities found in new scan for site: #{site_id}.") if empty_report
           log_message("Old vulnerabilities found in new scan for site #{site_id}, preparing closures.") unless empty_report
           unless empty_report
-            tickets = helper.prepare_close_tickets(old_scan_vuln)
-            helper.close_tickets(tickets)
+            ticket_rate_limiter(options, old_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_close_tickets(ticket_batch)}, Proc.new {|tickets| helper.close_tickets(tickets)})
           end
         else
           # Create a log message but do not halt execution of the helper if ticket closeing is not 
           # supported to allow legacy code to execute normally.
-          log_message("Helper does not impelment close methods.")
+          log_message('Helper does not impelment close methods.')
+        end
+      end
+    end
+
+    def ticket_rate_limiter(options, query_results_file, ticket_prepare_method, ticket_send_method)
+      batch_size_max = (options[:batch_size] + 1)
+      log_message("Batching tickets in sizes: #{options[:batch_size]}")
+
+      # Start the batching
+      query_results_file.rewind
+      csv_header = query_results_file.readline
+      ticket_batch = []
+      current_ip = -1
+      current_csv_row = nil
+
+      begin
+        IO.foreach(query_results_file) do |line|
+          ticket_batch << line
+
+          CSV.parse(line.chomp, headers: csv_header)  do |row|
+            if current_ip == -1
+              current_ip = row['ip_address']  unless row['ip_address'] == 'ip_address'
+            end
+            current_csv_row = row unless row['ip_address'] == 'ip_address'
+          end
+
+          if ticket_batch.size >= batch_size_max
+            #Batch target reached. Make sure we  end with a complete IP address set (all tickets for a single IP in this batch)
+            if(current_ip != current_csv_row['ip_address'])
+              log_message('Batch size reached. Sending tickets.')
+
+              #Move the mismatching line to the next batch
+              line_holder = ticket_batch.pop
+              ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)
+              # Cleanup for the next batch
+              ticket_batch.clear
+              ticket_batch << csv_header
+              ticket_batch << line_holder
+              current_ip = -1
+            end
+          end
         end
+      ensure
+        log_message('Finished reading report. Sending any remaining tickets and cleaning up file system.')
+        ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)
+        query_results_file.close
+        query_results_file.unlink
+      end
+    end
+
+    def ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)
+      #Just the header (no tickets).
+      if ticket_batch.size == 1
+        log_message('Received empty batch. Not sending tickets.')
+        return
       end
+
+      # Prep the batch of tickets
+      log_message('Creating tickets.')
+      tickets = ticket_prepare_method.call(ticket_batch.join(''))
+      log_message("Generated tickets: #{ticket_batch.size}")
+      # Sent them off
+      log_message('Sending tickets.')
+      ticket_send_method.call(tickets)
+      log_message('Returning for next batch.')
     end
 
+
     # Starts the Ticketing Service.
     def start
       # Checks if the csv historical file already exists && reads it, otherwise create it && assume first time run.
       file_site_histories = prepare_historical_data(@ticket_repository, @options)
-      # If we didn't specify a site || first time run, then it gets all the vulnerabilities.
-      if @options[:sites].empty? || @first_time
+      historical_scan_file = File.join(File.dirname(__FILE__), "#{@options[:file_name]}")
+      # If we didn't specify a site || first time run (no scan history), then it gets all the vulnerabilities.
+      if @options[:sites].empty? || file_site_histories.nil?
+        log_message('Storing current scan state before obtaining all vulnerabilities.')
+        current_scan_state = ticket_repository.load_last_scans(options)
+
         all_site_report(@ticket_repository, @options, @helper)
+
+        #Generate historical CSV file after completing the fist query.
+        log_message('No historical CSV file found. Generating.')
+        @ticket_repository.save_scans_to_file(historical_scan_file, current_scan_state)
+        log_message('Historical CSV file generated.')
       else
         log_message('Obtaining last scan information.')
-        @nexpose_site_histories = @ticket_repository.last_scans
+        @nexpose_site_histories = @ticket_repository.last_scans(@options)
+
+        # Scan states can change during our processing. Store the state we are
+        # about to process and move this to the historical_scan_file if we
+        # successfully process.
+        log_message('Calculated deltas, storing current scan state.')
+        current_scan_state = ticket_repository.load_last_scans(options)
+
         # Only run if a scan has been ran ever in Nexpose.
         unless @nexpose_site_histories.empty?
           delta_site_report(@ticket_repository, @options, @helper, file_site_histories)
+          # Processing completed successfully. Update historical scan file.
+          @ticket_repository.save_scans_to_file(historical_scan_file, current_scan_state)
         end
       end
+      log_message('Exiting ticket service.')
     end
   end
 end

metadata

@@ -1,41 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: nexpose_ticketing
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Damian Finol
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-08 00:00:00.000000000 Z
+date: 2014-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nexpose
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: savon
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
 description: This gem provides a Ruby implementation of different integrations with
@@ -67,9 +67,9 @@ files:
 - lib/nexpose_ticketing/helpers/servicenow_helper.rb
 - lib/nexpose_ticketing/nx_logger.rb
 - lib/nexpose_ticketing/queries.rb
+- lib/nexpose_ticketing/report_helper.rb
 - lib/nexpose_ticketing/ticket_repository.rb
 - lib/nexpose_ticketing/ticket_service.rb
-- nexpose_ticketing.gemspec
 homepage: https://github.com/rapid7/nexpose_ticketing
 licenses:
 - BSD
@@ -80,17 +80,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.4
 signing_key: 
 specification_version: 4
 summary: Ruby Nexpose Ticketing Engine.

data/nexpose_ticketing.gemspec

@@ -1,20 +0,0 @@
-# encoding: utf-8
-
-Gem::Specification.new do |s|
-  s.name                  = 'nexpose_ticketing'
-  s.version               = '0.2.3'
-  s.homepage              = 'https://github.com/rapid7/nexpose_ticketing'
-  s.summary               = 'Ruby Nexpose Ticketing Engine.'
-  s.description           = 'This gem provides a Ruby implementation of different integrations with ticketing services for Nexpose.'
-  s.license               = 'BSD'
-  s.authors               = ['Damian Finol']
-  s.email                 = ['damian_finol@rapid7.com']
-  s.files                 = Dir['[A-Z]*'] + Dir['lib/**/*'] + Dir['tests/**']
-  s.require_paths         = ['lib']
-  s.extra_rdoc_files      = ['README.md']
-  s.required_ruby_version = '>= 1.9'
-  s.platform              = 'ruby'
-  s.executables           = ['nexpose_jira','nexpose_servicenow','nexpose_remedy']
-  s.add_dependency('nexpose', '>= 0.6.0')
-  s.add_dependency('savon', '~> 2.1')
-end