nexpose_servicenow 0.4.24 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3df03f3e1e1368fc16c60c61da775e397035309f
-  data.tar.gz: 5997f1479b0084bb174329a43ac92373240ec704
+  metadata.gz: 7f3da5fa2194d025ab5192815f682a09a94583be
+  data.tar.gz: 94e1f6a2c0976fba3215185d62a2f0a03edcb5c5
 SHA512:
-  metadata.gz: ad9e10d478b454d5b474c20f6cfdddb8018f979f71562f0948a569f76ef6e1ee0c9dfa3ad871013d6b4f3cf5d72bc3c9aa33bb016b0226f219976137a06cf16c
-  data.tar.gz: 7c07adc6765b58543b9172b3bb0b00ab01f9fa6e6fc21698a6b07c02681cca728fba16843e200c4d7ef1ae2ad7130778d38ddd7f98a631d5228aabc55f7c9962
+  metadata.gz: 6c6223fa14d6e6b0221e6176f5a4b79c4e9816dc2350592b68262cc94a98cf2cdf26d3bc647f699102c20954f9120fa12fdb1d0d4aa041a9c8cda75748036322
+  data.tar.gz: b5f07c1718b8cf959b932d6d76192c8ccd2a71b30ec70805334ab117f2e4f3b0e373ac2c3e876dced59f2e2d7f1dabc427568dd547c854cb636f6700c2b1fb2b

data/README.md

@@ -1,14 +1,22 @@
 # NexposeServicenow
 
 ## Installation
+The gem may installed via the following command from the RubyGems repository:
 
-gem install nexpose_servicenow
+`gem install nexpose_servicenow`
 
 ## Usage
+The gem is called by the ServiceNow console when a vulnerability integration executes.
 
-## Development
+Alternatively, it is also possible to call the following to see a list of parameters:
+`nexpose_servicenow -h`
 
-## Contributing
+
+## Support
+Please contact the following address for support queries:
+[support@rapid7.com](support@rapid7.com) 
+
+Please attach both the gem logs and relevant snippets from the agent logs.
 
 ## License
 

data/lib/nexpose_servicenow.rb

@@ -8,31 +8,22 @@ require 'nexpose_servicenow/arg_parser'
 require 'nexpose_servicenow/chunker'
 require 'nexpose_servicenow/nx_logger'
 require 'nexpose_servicenow/historical_data'
-require "nexpose_servicenow/version"
+require 'nexpose_servicenow/version'
 
 module NexposeServiceNow
   class Main
     def self.start(args)
       options = ArgParser.parse(args)
 
-      log = setup_logging(options)
-      
-      censored_options = options.dup
-      censored_options[:nexpose_username] = "*****"
-      censored_options[:nexpose_password] = "*****"
-      log.log_message("Options: #{censored_options}")
+      @log = setup_logging(options)
 
-      if options[:mode].to_s == ""
-        log.log_message("Script was called without mode.")
-        puts "No mode selected. Use -h to see command line options."
-        exit -1
-      end
+      censored_options = options.dup
+      censored_options[:nexpose_username] = '*****'
+      censored_options[:nexpose_password] = '*****'
+      @log.log_message("Options: #{censored_options}")
 
-      if options[:nexpose_ids].first.to_s == "0"
-        log.log_error_message('Retrieving array of all site IDs')
-        options[:nexpose_ids] = get_nexpose_helper(options).all_sites.sort
-        options[:nexpose_ids].map! { |i| i.to_s }
-      end             
+      options[:nexpose_ids] = get_collection_ids(options)
+      options[:start_time] = Time.new().strftime('%Y-%m-%m %H:%M:%S')
 
       report_details = NexposeHelper.get_report_names(options[:query], 
                                                     options[:nexpose_ids])
@@ -41,13 +32,21 @@ module NexposeServiceNow
                                                      options[:output_dir])
       end
 
-      update_last_scan_data(options)
-      create_report(report_details, options)  
+      update_delta_files(options) if options[:mode] == 'latest_scans'
 
-      log.log_message("Initialising #{options[:mode]} mode")
+      create_report(report_details, options)
+
+      @log.log_message("Initialising #{options[:mode]} mode")
       self.send("#{options[:mode]}_mode", report_details, options)
     end
 
+    def self.get_historical_data(options)
+      HistoricalData.new(options[:output_dir],
+                         options[:nexpose_ids],
+                         options[:id_type],
+                         options[:start_time])
+    end
+
     def self.get_nexpose_helper(options)
       NexposeHelper.new(options[:nexpose_url], 
                         options[:nexpose_port],
@@ -55,6 +54,15 @@ module NexposeServiceNow
                         options[:nexpose_password])
     end
 
+    # Retrieves list of all IDs if the user has chosen to import each group
+    def self.get_collection_ids(options)
+      return options[:nexpose_ids] if options[:nexpose_ids].first.to_s != '0'
+
+      @log.log_error_message("Retrieving array of all #{options[:id_type]} IDs")
+      helper = get_nexpose_helper(options)
+      helper.collection_ids(options[:id_type]).map(&:to_s)
+    end
+
     def self.setup_logging(options)
       log = NexposeServiceNow::NxLogger.instance
       log.setup_statistics_collection(NexposeServiceNow::VENDOR, 
@@ -66,43 +74,42 @@ module NexposeServiceNow
       log
     end
 
-    #Merges in the details from the last time the 
-    #integration ran reports.
-    def self.update_last_scan_data(options)
-      return unless options[:mode] == "latest_scans"
-      historical_data = HistoricalData.new(options)
-      historical_data.update_last_scan_data
-
+    # Merges in the details from the last time the integration ran reports.
+    def self.update_delta_files(options)
+      historical_data = get_historical_data(options)
+      historical_data.update_delta_file
       historical_data.save_vuln_timestamp(filter_sites(options))
     end
 
-    #Create a report if explicitly required or else an existing
-    #report file isn't found
+    # Create a report if explicitly required or else an existing
+    # report file isn't found
     def self.create_report(report_details, options)
-      return if options[:mode].start_with? 'update_'
+      if options[:mode].start_with? 'update_' or
+             (options[:mode] == 'latest_scans' && options[:id_type] != :site)
+        return
+      end
 
       unless options[:gen_report]
-        #If any file is missing, perform all queries
-        return if report_details.all? { |f| File.exists?(f[:report_name]) } 
+        # If any file is missing, perform all queries
+        return if report_details.all? { |f| File.exists? f[:report_name] }
       end
 
       credentials = %i{nexpose_username nexpose_password}
-      if credentials.any? { |cred| options[cred].to_s == "" }
-        log = NexposeServiceNow::NxLogger.instance
-        log.log_error_message "Nexpose credentials necessary but not supplied."
+      if credentials.any? { |cred| options[cred].to_s == '' }
+        @log.log_error_message 'Nexpose credentials necessary but not supplied.'
         exit -1
       end
 
       #Filter it down to sites which actively need queried
       sites_to_scan = filter_sites(options)
       nexpose_helper = get_nexpose_helper(options)
-      hist_data = HistoricalData.new(options)
+      hist_data = get_historical_data(options)
       vuln_query = options[:query].to_s.start_with? 'vulnerabili'
-      last_run = nil
-      
-      start_time = Time.new
-      query_options = { last_scans: hist_data.last_scan_ids(sites_to_scan) }      
+
+      delta_values = hist_data.stored_delta_values(sites_to_scan)
+      query_options = { delta_values: delta_values }
       query_options[:vuln_query_date] = hist_data.last_vuln_run if vuln_query
+      query_options[:filters] = options[:filters]
 
       filename = nexpose_helper.create_report(options[:query],
                                               sites_to_scan,
@@ -110,8 +117,8 @@ module NexposeServiceNow
                                               options[:output_dir],
                                               query_options)
 
-      #A single String may be returned or an Array of Strings
-      if filename.class.to_s == "Array"
+      # A single String may be returned or an Array of Strings
+      if filename.class.to_s == 'Array'
         filename.map! { |f| File.expand_path(options[:output_dir], f) }
         return filename.join("\n")
       end
@@ -120,8 +127,8 @@ module NexposeServiceNow
     end
 
     def self.filter_sites(options)
-      #These queries always run to make sure certain data is up to date
-      exceptions = ['vulnerabili', 'asset_groups', 'sites', 'tags']
+      # These queries always run to make sure certain data is up to date
+      exceptions = %w(vulnerabili asset_groups sites tags)
       if exceptions.any? { |e| options[:query].to_s.start_with? e }
         return options[:nexpose_ids] 
       end
@@ -132,28 +139,30 @@ module NexposeServiceNow
         return options[:nexpose_ids] 
       end
 
-      historical_data = HistoricalData.new(options)
+      historical_data = get_historical_data(options)
       imported_sites_only = options[:query].to_s.eql? 'vulnerable_old_items'
-      sites_to_scan = historical_data.sites_to_scan(imported_sites_only)
+      sites_to_scan = historical_data.collections_to_import(imported_sites_only)
 
       return sites_to_scan unless (sites_to_scan.nil? || sites_to_scan.empty?)
 
-      log = NexposeServiceNow::NxLogger.instance
-      log.log_message "Sites #{options[:nexpose_ids]} are up to date."
-      log.log_message "Query requested was: #{options[:query]}."
+      @log.log_message "Sites #{options[:nexpose_ids]} are up to date."
+      @log.log_message "Query requested was: #{options[:query]}."
       exit 0
     end
 
-    #Print the chunk info
+    # Print the chunk info
     def self.chunk_info_mode(report_details, options)
       filtered_sites = filter_sites(options)
-      report_details = report_details.select { |d| d[:id] == -1 or filtered_sites.include? d[:id] }
+      report_details = report_details.select do |d|
+        d[:id] == -1 or filtered_sites.include? d[:id]
+      end
       chunker = Chunker.new(report_details, options[:row_limit])
 
-      puts chunker.preprocess(filtered_sites)
+      # TODO: Check why filtered_sites are passed in here
+      puts chunker.preprocess
     end
 
-    #Prints a chunk of CSV to the console
+    # Prints a chunk of CSV to the console
     def self.get_chunk_mode(report_details, options)
       #Get the byte offset and length
       chunker = Chunker.new(report_details, options[:row_limit])
@@ -165,28 +174,28 @@ module NexposeServiceNow
     end
 
     def self.latest_scans_mode(report_details, options)
-      historical_data = HistoricalData.new(options)
+      historical_data = get_historical_data(options)
       puts historical_data.filter_report
     end
 
     def self.remove_last_scan_mode(report_details, options)
-      historical_data = HistoricalData.new(options)
+      historical_data = get_historical_data(options)
       historical_data.remove_last_scan_data
     end
 
     def self.update_last_scan_mode(report_details, options)
-      historical_data = HistoricalData.new(options)
+      historical_data = get_historical_data(options)
       historical_data.set_last_scan(options[:nexpose_ids].first,
                                     options[:last_scan_data])
     end
     
     def self.remove_last_vuln_mode(report_details, options)
-      historical_data = HistoricalData.new(options)
+      historical_data = get_historical_data(options)
       historical_data.remove_last_vuln_data
     end
     
     def self.update_last_vuln_mode(report_details, options)
-      historical_data = HistoricalData.new(options)
+      historical_data = get_historical_data(options)
       historical_data.set_last_vuln(options[:last_scan_data],
                                     options[:nexpose_ids])
     end

data/lib/nexpose_servicenow/arg_parser.rb

@@ -1,119 +1,211 @@
 require 'optparse'
 require 'json'
+require 'time'
 require_relative './queries'
 require_relative './nx_logger'
 
 module NexposeServiceNow
   class ArgParser
-    NX_ID_TYPES = %i[site tag]
+    NX_ID_TYPES = %i[site asset_group]
     MODES = %i[chunk_info get_chunk latest_scans
                remove_last_scan update_last_scan
                remove_last_vuln update_last_vuln]
 
     QUERY_NAMES = Queries.methods(false)
-    
-    QUERY_ALIASES = { 'devices' => 'cmdb_ci_outofband_device', 
-                      'vuln_items' => 'sn_vul_vulnerable_item', 
-                      'vuln_entries' => 'sn_vul_third_party_entry' }
-     
+
     def self.parse(args)
       options = Hash.new
 
+      log = NexposeServiceNow::NxLogger.instance
+      log.log_message 'Parsing options.'
+
       opt_parser = OptionParser.new do |opts|
-        opts.banner = "Usage: example.rb [options]"
+        opts.banner = 'Usage: example.rb [options]'
 
-        opts.on("-o", "--output-dir DIRECTORY",
-                "Directory in which to save reports") do |output_dir|
+        opts.on('-o', '--output-dir DIRECTORY',
+                'Directory in which to save reports') do |output_dir|
           options[:output_dir] = output_dir
         end
 
-        opts.on("-m", "--mode MODE",
+        opts.on('-m', '--mode MODE',
                 "Mode for program output. (#{MODES.join(', ')})") do |mode|
           options[:mode] = mode
         end
 
-        opts.on("-g", "--generate-report BOOLEAN",
-                "True to generate and download new report.") do |gen|
+        opts.on('-g', '--generate-report BOOLEAN',
+                'True to generate and download new report') do |gen|
           char = gen.downcase[0]
-          options[:gen_report] = char == 'y' || char == 't'
+          options[:gen_report] = %w(y t).any? { |c| c == char }
         end
 
-        opts.separator ""
-        opts.separator "Query options:"
+        opts.separator ''
+        opts.separator 'Query options:'
 
-        opts.on("-q", "--query QUERY", QUERY_NAMES, 
+        opts.on('-q', '--query QUERY', QUERY_NAMES,
                 "Select query (#{QUERY_NAMES.join(', ')})") do |query| 
           options[:query] = query
         end
 
-        opts.on("-t", "--type TYPE", NX_ID_TYPES, 
+        opts.on('-t', '--type TYPE', NX_ID_TYPES,
                 "Select type (#{NX_ID_TYPES.join(', ')})") do |type| 
           options[:id_type] = type
         end
 
-        opts.on("-i", "--items x,y,z", Array, 
-                "IDs of the nexpose items to scan") do |items|
+        opts.on('-i', '--items x,y,z', Array,
+                'IDs of the nexpose items to scan') do |items|
           options[:nexpose_ids] = items
         end
 
-        opts.separator ""
-        opts.separator "Nexpose options:"
+        opts.separator ''
+        opts.separator 'Nexpose options:'
         
-        opts.on("-n", "--nexpose-address URL", 
-                "URL of the Nexpose server") do |url|
+        opts.on('-n', '--nexpose-address URL',
+                'URL of the Nexpose server') do |url|
           port = url.slice!(/:(\d+)$/)
           port.slice! ':' unless port.nil?
 
-          url.slice! /https:\/\//
+          url.slice! 'https://'
           options[:nexpose_url] = url
           options[:nexpose_port] = port || '3780'
-          @full_url = options[:nexpose_url] + ':' + options[:nexpose_port]
         end
 
-        opts.on("-u", "--user USER",
-                "Username for Nexpose console") do |username|
+        opts.on('-u', '--user USER',
+                'Username for Nexpose console') do |username|
           options[:nexpose_username] = username
         end
 
-        opts.on("-p", "--password PASSWORD",
-                "Password for the Nexpose user") do |password|
+        opts.on('-p', '--password PASSWORD',
+                'Password for the Nexpose user') do |password|
           options[:nexpose_password] = password
         end
 
-        opts.separator ""
-        opts.separator "Chunk info mode options:"
+        opts.separator ''
+        opts.separator 'Chunk info mode options:'
         
-        opts.on("-r", "--row-limit LIMIT",
-                "Maximum number of rows per chunk (with header).") do |limit|
+        opts.on('-r', '--row-limit LIMIT',
+                'Maximum number of rows per chunk (including header).') do |limit|
           options[:row_limit] = limit.to_i
           options[:row_limit] = 9_999_999 if options[:row_limit] <= 0
         end
 
-        opts.separator ""
-        opts.separator "Get chunk mode options:"
+        opts.separator ''
+        opts.separator 'Get chunk mode options:'
 
-        opts.on("-s", "--start START",
-                "The chunk starting offset.") do |start|
+        opts.on('-s', '--start START',
+                'The chunk starting offset.') do |start|
           options[:chunk_start] = start.to_i
         end
 
-        opts.on("-l", "--length LENGTH",
-                "The chunk length.") do |length|
+        opts.on('-l', '--length LENGTH',
+                'The chunk length.') do |length|
           options[:chunk_length] = length.to_i
         end
 
-        opts.separator ""
-        opts.separator "Last scan file modification options:"
+        opts.separator ''
+        opts.separator 'Filter options:'
+
+=begin
+        # CVE filter is not currently supported.
+        opts.on('-v', '--vuln-identifier CVE',
+                'The CVE values for which to filter.') do |data|
+          # A value of 'none' means the user has left field blank
+          if data.to_s.downcase != 'none'
+            data = data.to_s.sub(' ', '').split(',')
+
+            invalid_cve = data.select { |f| (f =~ /(CVE-)?\d{4}-\d+/) == nil }
+            unless invalid_cve.empty?
+              error = "Invalid CVEs applied: #{invalid_cve}"
+              puts error
+              log.log_message error
+              exit -1
+            end
+
+            data = data.map do |c|
+              if c.start_with? 'CVE-'
+                c
+              else
+                "CVE-#{c}"
+              end
+            end
+          else
+            data = nil
+          end
+
+          options[:filters] ||= {}
+          options[:filters][:cve] = data
+        end
+=end
+        opts.on('-c', '--cvss-score CVSS',
+                'The minimum CVSS score to import') do |data|
+
+          cvss_range = data.split('~')
+
+          if cvss_range.count != 2
+            error = "Expected two CVSS scores. Received #{cvss_range.count}"
+            puts error
+            log.log_message error
+            exit -1
+          end
+
+          cvss_range.each do |cvss|
+            next if cvss.to_s =~ /^0*(10(\.0+)?|\d(\.\d+)?)?$/
+            error = "Invalid CVSS score supplied: #{cvss}. Exiting"
+            puts error
+            log.log_message error
+            exit -1
+          end
+
+          options[:filters] ||= {}
+          options[:filters][:cvss] = cvss_range
+        end
 
-        opts.on("-d", "--data DATA",
-                "Date or scan ID to be inserted in last scan file.") do |data|
+        opts.on('-d', '--date DATE',
+                'The minimum date for each vulnerability instance.') do |date|
+          # Date should be in format 'YYYY-MM-DD~YYYY-MM-DD'
+          dates = date.to_s.split('~')
+
+          if dates.count != 2
+            error = "Expected two dates. Received #{dates.count}"
+            puts error
+            log.log_message error
+            exit -1
+          end
+
+          # Add the dates
+          dates[0] = dates[0] + ' 00:00:00'
+          dates[1] = dates[1] + ' 23:59:59'
+
+          # Check for valid dates and placeholders
+          dates.map! do |d|
+            if d =~ /Y{4}-M{1,2}-D{1,2}/i
+              nil
+            elsif d =~ /\d{4}-\d{1,2}-\d{1,2}/
+              d
+            else
+              error = "Invalid date supplied: #{d}. Exiting."
+              puts error
+              log.log_message error
+              exit -1
+            end
+          end
+
+          options[:filters] ||= {}
+          options[:filters][:date] = dates
+        end
+
+
+        opts.separator ''
+        opts.separator 'Last scan file modification options:'
+
+        opts.on('-e', '--errata DATA',
+                'Date or scan ID to be inserted in last scan file.') do |data|
           options[:last_scan_data] = data
         end
 
-        opts.separator ""
-        opts.separator "Common options:"
+        opts.separator ''
+        opts.separator 'Common options:'
 
-        opts.on_tail("-h", "--help", "Show this message") do
+        opts.on_tail('-h', '--help', 'Show this message') do
           puts opts
           exit
         end
@@ -125,21 +217,27 @@ module NexposeServiceNow
       options
     end
 
-    #TODO: Validate input depending on mode AND whether generating a report
-    # is required.
     def self.validate_input(options)
       #Insert defaults. Some are mode-specific.
       options[:output_dir] ||= '.'
       options[:row_limit] ||= 9_999_999
       options[:id_type] ||= 'site'
       options[:nexpose_ids] ||= []
+      options[:filters] ||= {}
 
       options[:query] = 'latest_scans' if options[:mode] == 'latest_scans'
 
       #By default, a report won't be generated if a chunk's being retrieved
       if options[:gen_report].nil?
-        options[:gen_report] = options[:mode] == "chunk_info" ||
-                               options[:mode] == "latest_scans"
+        options[:gen_report] = options[:mode] == 'chunk_info' ||
+                               options[:mode] == 'latest_scans'
+      end
+
+      if options[:mode].to_s == ''
+        log = NexposeServiceNow::NxLogger.instance
+        log.log_message('Script was called without mode.')
+        puts 'No mode selected. Use -h to see command line options.'
+        exit -1
       end
 
       options
@@ -150,7 +248,7 @@ module NexposeServiceNow
       return options if options[:gen_report] == false
 
       log = NexposeServiceNow::NxLogger.instance
-      log.log_message "Retrieving environment variables."
+      log.log_message 'Retrieving environment variables.'
 
       # Retrieve environment variable settings
       %i[url port username password].each do |setting|