nexpose_servicenow 0.6.2 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 58a608ff53a461dc491abb88f3a42b0043dc38e2
-  data.tar.gz: fadea12c81f970626257ae7aef396a4a7050c25e
+  metadata.gz: ba87a2439d290abbf57e7eb2d6792d691166f312
+  data.tar.gz: 0dcb0dc8fc8373431a3ccba7b6879903562a0fee
 SHA512:
-  metadata.gz: c326787fa3fded2fc9db9adaa69552a8e4a7c3849865da757ac3735d4540b041d6afe6efa33ca6c708833e61872cd52452569ece6e58ff35dd5d7dd625bd9c49
-  data.tar.gz: 4651506f36ae3fea4f5842f716b57c270cb44f37a0465820294aee3730a859c0f0731ee57481c9afe5fcbbaf71804e0d845ab016db761be15c9384e169f0d4fe
+  metadata.gz: df8bcf8ccf412d0fe87aa41282233fc6e9739ccac94ec7398c42bd73842ec5de92ef7045ae300ad0e47999c808e5c327caf8fe974997066f349dfef54c45de63
+  data.tar.gz: 2f8dfbd40ea34d981d75da8b0bad547de128b4e3cd118c9bf3dec4883aeab01a61664f29d37a4b1156d60326864874fd3aa0e6a7d929510e356660cb425a2b76

data/README.md

@@ -14,7 +14,7 @@ Alternatively, it is also possible to call the following to see a list of parame
 
 ## Support
 Please contact the following address for support queries:
-[support@rapid7.com](support@rapid7.com) 
+[Rapid7 Support Portal](https://rapid7support.force.com/customers/login) 
 
 Please attach both the gem logs and relevant snippets from the agent logs.
 

data/lib/nexpose_servicenow.rb

@@ -2,8 +2,9 @@ require 'csv'
 require 'optparse'
 require 'nexpose'
 require 'uri'
-require 'nexpose_servicenow/queries'
-require 'nexpose_servicenow/nexpose_helper'
+require_relative './nexpose_servicenow/helpers/connection_helper'
+require_relative './nexpose_servicenow/helpers/nexpose_console_helper'
+require_relative './nexpose_servicenow/helpers/data_warehouse_helper'
 require 'nexpose_servicenow/arg_parser'
 require 'nexpose_servicenow/chunker'
 require 'nexpose_servicenow/nx_logger'
@@ -18,49 +19,54 @@ module NexposeServiceNow
       @log = setup_logging(options)
 
       censored_options = options.dup
-      censored_options[:nexpose_username] = '*****'
-      censored_options[:nexpose_password] = '*****'
+      censored_options[:username] = '*****'
+      censored_options[:password] = '*****'
       @log.log_message("Options: #{censored_options}")
 
-      options[:nexpose_ids] = get_collection_ids(options)
-      options[:start_time] = Time.new().strftime('%Y-%m-%m %H:%M:%S')
+      query = options[:query]
+      site_ids = options[:nexpose_ids]
+
+      # Filter out irrelevant sites
+      if query == :vulnerable_old_items
+        site_ids = get_historical_data(options).filter_ids(site_ids)
+        if site_ids.count == 0
+          puts 'No sites remaining for vulnerable old items query. Exiting.'
+          exit 0
+        end
+      end
+
+      report_details = ConnectionHelper.get_report_names(query, site_ids)
 
-      report_details = NexposeHelper.get_report_names(options[:query], 
-                                                    options[:nexpose_ids])
       report_details.each do |r|
-        r[:report_name] = NexposeHelper.get_filepath(r[:report_name], 
-                                                     options[:output_dir])
+        r[:report_name] = ConnectionHelper.get_filepath(r[:report_name],
+                                                        options[:output_dir])
       end
 
-      update_delta_files(options) if options[:mode] == 'latest_scans'
+      report_results = create_report(report_details, options)
 
-      create_report(report_details, options)
+      # If data was returned, we can short circuit here
+      if !report_results.nil? and options[:mode] == 'chunk_info'
+        puts report_results
+        exit 0
+      end
 
       @log.log_message("Initialising #{options[:mode]} mode")
       self.send("#{options[:mode]}_mode", report_details, options)
     end
 
     def self.get_historical_data(options)
-      HistoricalData.new(options[:output_dir],
-                         options[:nexpose_ids],
-                         options[:id_type],
-                         options[:start_time])
-    end
-
-    def self.get_nexpose_helper(options)
-      NexposeHelper.new(options[:nexpose_url], 
-                        options[:nexpose_port],
-                        options[:nexpose_username],
-                        options[:nexpose_password])
+      HistoricalData.new(options[:output_dir])
     end
 
-    # Retrieves list of all IDs if the user has chosen to import each group
-    def self.get_collection_ids(options)
-      return options[:nexpose_ids] if options[:nexpose_ids].first.to_s != '0'
+    def self.get_helper(options)
+      name = options[:conn_type].to_s.split('_').map(&:capitalize).join('')
 
-      @log.log_error_message("Retrieving array of all #{options[:id_type]} IDs")
-      helper = get_nexpose_helper(options)
-      helper.collection_ids(options[:id_type]).map(&:to_s)
+      helper = eval("#{name}Helper")
+      helper.new(options[:url],
+                 options[:port],
+                 options[:username],
+                 options[:password],
+                 options[:database_name])
     end
 
     def self.setup_logging(options)
@@ -74,13 +80,6 @@ module NexposeServiceNow
       log
     end
 
-    # Merges in the details from the last time the integration ran reports.
-    def self.update_delta_files(options)
-      historical_data = get_historical_data(options)
-      historical_data.update_delta_file
-      historical_data.save_vuln_timestamp(filter_sites(options))
-    end
-
     # Create a report if explicitly required or else an existing
     # report file isn't found
     def self.create_report(report_details, options)
@@ -89,76 +88,45 @@ module NexposeServiceNow
         return
       end
 
+      # Perform all queries if a file is missing, regardless of other settings
       unless options[:gen_report]
-        # If any file is missing, perform all queries
         return if report_details.all? { |f| File.exists? f[:report_name] }
       end
 
-      credentials = %i{nexpose_username nexpose_password}
+      credentials = %i{username password}
       if credentials.any? { |cred| options[cred].to_s == '' }
         @log.log_error_message 'Nexpose credentials necessary but not supplied.'
         exit -1
       end
 
-      #Filter it down to sites which actively need queried
-      sites_to_scan = filter_sites(options)
-      nexpose_helper = get_nexpose_helper(options)
-      hist_data = get_historical_data(options)
-      vuln_query = options[:query].to_s.start_with? 'vulnerabili'
+      # Filter it down to sites which actively need queried
+      sites_to_scan = options[:nexpose_ids].keys
 
-      delta_values = hist_data.stored_delta_values(sites_to_scan)
-      query_options = { delta_values: delta_values }
-      query_options[:vuln_query_date] = hist_data.last_vuln_run if vuln_query
+      query_options = { delta_values: options[:nexpose_ids] }
+      query_options[:vuln_query_date] = options[:vuln_query_date]
       query_options[:filters] = options[:filters]
-
-      filename = nexpose_helper.create_report(options[:query],
-                                              sites_to_scan,
-                                              options[:id_type],
-                                              options[:output_dir],
-                                              query_options)
-
-      # A single String may be returned or an Array of Strings
-      if filename.class.to_s == 'Array'
-        filename.map! { |f| File.expand_path(options[:output_dir], f) }
-        return filename.join("\n")
-      end
-
-      File.expand_path(options[:output_dir], filename)
-    end
-
-    def self.filter_sites(options)
-      # These queries always run to make sure certain data is up to date
-      exceptions = %w(vulnerabili asset_groups sites tags)
-      if exceptions.any? { |e| options[:query].to_s.start_with? e }
-        return options[:nexpose_ids] 
-      end
-
-      #Always run the query for latest scans or vulnerabilities
-      if options[:mode] == 'latest_scans' || 
-         options[:mode] == 'get_chunk'
-        return options[:nexpose_ids] 
-      end
-
-      historical_data = get_historical_data(options)
-      imported_sites_only = options[:query].to_s.eql? 'vulnerable_old_items'
-      sites_to_scan = historical_data.collections_to_import(imported_sites_only)
-
-      return sites_to_scan unless (sites_to_scan.nil? || sites_to_scan.empty?)
-
-      @log.log_message "Sites #{options[:nexpose_ids]} are up to date."
-      @log.log_message "Query requested was: #{options[:query]}."
-      exit 0
+      query_options[:page_size] = options[:row_limit]
+      query_options[:row_limit] = options[:row_limit]
+      report_helper = get_helper(options)
+      @log.log_message("Querying using the #{report_helper.class}.")
+      report_helper.generate_report(options[:query],
+                                     sites_to_scan,
+                                     options[:id_type],
+                                     options[:output_dir],
+                                     query_options)
     end
 
     # Print the chunk info
     def self.chunk_info_mode(report_details, options)
-      filtered_sites = filter_sites(options)
+      site_ids = options[:nexpose_ids].keys
+
+      # Assign -1 to reports without site IDs
       report_details = report_details.select do |d|
-        d[:id] == -1 or filtered_sites.include? d[:id]
+        d[:id] == -1 or site_ids.include? d[:id]
       end
+
       chunker = Chunker.new(report_details, options[:row_limit])
 
-      # TODO: Check why filtered_sites are passed in here
       puts chunker.preprocess
     end
 
@@ -166,16 +134,15 @@ module NexposeServiceNow
     def self.get_chunk_mode(report_details, options)
       #Get the byte offset and length
       chunker = Chunker.new(report_details, options[:row_limit])
-      filtered_sites = filter_sites(options)
 
       puts chunker.read_chunk(options[:chunk_start],
                               options[:chunk_length],
-                              filtered_sites.first)
+                              options[:nexpose_ids].keys.first)
     end
 
     def self.latest_scans_mode(report_details, options)
       historical_data = get_historical_data(options)
-      puts historical_data.filter_report
+      puts historical_data.filter_report options[:nexpose_ids].keys
     end
 
     def self.remove_last_scan_mode(report_details, options)
@@ -202,7 +169,7 @@ module NexposeServiceNow
 
     def self.remove_diff_comparison_mode(report_details, options)
       historical_data = get_historical_data(options)
-      historical_data.remove_last_diff_comparison_data options[:output_dir]
+      historical_data.remove_diff_files options[:output_dir]
     end
   end
 end

data/lib/nexpose_servicenow/arg_parser.rb

@@ -1,17 +1,17 @@
 require 'optparse'
 require 'json'
 require 'time'
-require_relative './queries'
+require_relative './queries/nexpose_queries'
 require_relative './nx_logger'
 
 module NexposeServiceNow
   class ArgParser
     NX_ID_TYPES = %i[site asset_group]
+    NX_CONNECTION_TYPES = %i[nexpose_console data_warehouse]
     MODES = %i[chunk_info get_chunk latest_scans
-               remove_last_scan update_last_scan
-               remove_last_vuln update_last_vuln]
-
-    QUERY_NAMES = Queries.methods(false)
+               remove_last_scan remove_last_vuln]
+    REQUIRED_OPTIONS = %i[url port username password]
+    QUERY_NAMES = NexposeQueries.methods(false)
 
     def self.parse(args)
       options = Hash.new
@@ -46,44 +46,71 @@ module NexposeServiceNow
           options[:query] = query
         end
 
-        opts.on('-t', '--type TYPE', NX_ID_TYPES,
-                "Select type (#{NX_ID_TYPES.join(', ')})") do |type| 
-          options[:id_type] = type
+        opts.on('-t', '--type ID~CONNECTION', 'Select ID type ' \
+                "(#{NX_ID_TYPES.join(', ')}) and connection type " \
+                "(#{NX_CONNECTION_TYPES.join(', ')})") do |types|
+          type = types.split('~')
+          options[:id_type] = type[0].intern
+          options[:conn_type] = type[1].intern
+
+          if options[:conn_type].equal? :data_warehouse
+            REQUIRED_OPTIONS << :database_name
+          end
         end
 
-        opts.on('-i', '--items x,y,z', Array,
-                'IDs of the nexpose items to scan') do |items|
-          options[:nexpose_ids] = items
+        opts.on('-i', '--items x~x,y~y,z~z', Array,
+                'IDs of the nexpose items to ' \
+                'scan, provided with their previous scan IDs or timestamp ' \
+                'of last scan') do |items|
+          options[:nexpose_ids] = {}
+
+          # Split the string up into site and scan pairs
+          items = items.map { |s| s.split('~') }
+
+          # Store the information in site:scan_id dict
+          items.each { |site, scan| options[:nexpose_ids][site] = scan }
+        end
+
+        opts.on('-a', '--abs-vulntime TIMESTAMP',
+                'Timestamp of last vulnerability definition import') do |vulnt|
+          # TODO: Does the date need formatted?
+          options[:vuln_query_date] = vulnt
         end
 
         opts.separator ''
-        opts.separator 'Nexpose options:'
+        opts.separator 'Connection options:'
         
-        opts.on('-n', '--nexpose-address URL',
-                'URL of the Nexpose server') do |url|
+        opts.on('-n', '--nexpose-datastore URL',
+                'URL of the Nexpose/Data Warehouse server') do |url|
           port = url.slice!(/:(\d+)$/)
           port.slice! ':' unless port.nil?
 
           url.slice! 'https://'
-          options[:nexpose_url] = url
-          options[:nexpose_port] = port || '3780'
+          options[:url] = url
+          options[:port] = port
         end
 
         opts.on('-u', '--user USER',
-                'Username for Nexpose console') do |username|
-          options[:nexpose_username] = username
+                'Username for Nexpose/Data Warehouse') do |username|
+          options[:username] = username
         end
 
         opts.on('-p', '--password PASSWORD',
-                'Password for the Nexpose user') do |password|
-          options[:nexpose_password] = password
+                'Password for the Nexpose/Data Warehouse user') do |password|
+          options[:password] = password
+        end
+
+        opts.on('-b', '--database DATABASE_NAME',
+                'The name of the Postgres Database '\
+                '(DataWarehouse Only)') do |database|
+          options[:database_name] = database
         end
 
         opts.separator ''
         opts.separator 'Chunk info mode options:'
         
         opts.on('-r', '--row-limit LIMIT',
-                'Maximum number of rows per chunk (including header).') do |limit|
+                'Maximum number of rows per chunk (inc. header).') do |limit|
           options[:row_limit] = limit.to_i
           options[:row_limit] = 9_999_999 if options[:row_limit] <= 0
         end
@@ -104,37 +131,7 @@ module NexposeServiceNow
         opts.separator ''
         opts.separator 'Filter options:'
 
-=begin
-        # CVE filter is not currently supported.
-        opts.on('-v', '--vuln-identifier CVE',
-                'The CVE values for which to filter.') do |data|
-          # A value of 'none' means the user has left field blank
-          if data.to_s.downcase != 'none'
-            data = data.to_s.sub(' ', '').split(',')
-
-            invalid_cve = data.select { |f| (f =~ /(CVE-)?\d{4}-\d+/) == nil }
-            unless invalid_cve.empty?
-              error = "Invalid CVEs applied: #{invalid_cve}"
-              puts error
-              log.log_message error
-              exit -1
-            end
-
-            data = data.map do |c|
-              if c.start_with? 'CVE-'
-                c
-              else
-                "CVE-#{c}"
-              end
-            end
-          else
-            data = nil
-          end
 
-          options[:filters] ||= {}
-          options[:filters][:cve] = data
-        end
-=end
         opts.on('-c', '--cvss-score CVSS',
                 'The minimum CVSS score to import') do |data|
 
@@ -175,6 +172,10 @@ module NexposeServiceNow
           dates[0] = dates[0] + ' 00:00:00'
           dates[1] = dates[1] + ' 23:59:59'
 
+          # Remove rogue '' in datetime format
+          dates[0] = dates[0].gsub("'","")
+          dates[1] = dates[1].gsub("'","")
+
           # Check for valid dates and placeholders
           dates.map! do |d|
             if d =~ /Y{4}-M{1,2}-D{1,2}/i
@@ -193,15 +194,6 @@ module NexposeServiceNow
           options[:filters][:date] = dates
         end
 
-
-        opts.separator ''
-        opts.separator 'Last scan file modification options:'
-
-        opts.on('-e', '--errata DATA',
-                'Date or scan ID to be inserted in last scan file.') do |data|
-          options[:last_scan_data] = data
-        end
-
         opts.separator ''
         opts.separator 'Common options:'
 
@@ -212,8 +204,8 @@ module NexposeServiceNow
       end 
 
       opt_parser.parse!(args)
-      options = self.validate_input(options)
       options = self.get_env_settings(options)
+      options = self.validate_input(options)
       options
     end
 
@@ -221,8 +213,10 @@ module NexposeServiceNow
       #Insert defaults. Some are mode-specific.
       options[:output_dir] ||= '.'
       options[:row_limit] ||= 9_999_999
+      options[:vuln_query_date] ||= '1985-01-01 00:00:00'
       options[:id_type] ||= 'site'
-      options[:nexpose_ids] ||= []
+      options[:conn_type] ||= :nexpose_console
+      options[:nexpose_ids] ||= {}
       options[:filters] ||= {}
 
       options[:query] = 'latest_scans' if options[:mode] == 'latest_scans'
@@ -233,36 +227,47 @@ module NexposeServiceNow
                                options[:mode] == 'latest_scans'
       end
 
+      options[:port] ||= if options[:conn_type].equal? :nexpose_console
+                           '3780'
+                         else
+                           '5432'
+                         end
+
+      log = NexposeServiceNow::NxLogger.instance
+
       if options[:mode].to_s == ''
-        log = NexposeServiceNow::NxLogger.instance
         log.log_message('Script was called without mode.')
         puts 'No mode selected. Use -h to see command line options.'
         exit -1
       end
 
+      #Only need to check these if a query is being performed
+      return options unless options[:gen_report]
+
+      REQUIRED_OPTIONS.each do |setting|
+        if options[setting].nil?
+          error = "Option #{setting} wasn't supplied."
+          log.log_error_message error
+          $stderr.puts "ERROR: #{error}"
+          exit -1
+        end
+      end
+
       options
     end
 
     def self.get_env_settings(options)
       #Only need these if a query is being performed
-      return options if options[:gen_report] == false
+      return options unless options[:gen_report]
 
       log = NexposeServiceNow::NxLogger.instance
       log.log_message 'Retrieving environment variables.'
 
       # Retrieve environment variable settings
-      %i[url port username password].each do |setting|
+      REQUIRED_OPTIONS.each do |setting|
         option = "nexpose_#{setting}"
-        setting = ENV[option.upcase]
-        sym = option.intern
-        options[sym] ||= setting
-
-        if options[sym].nil?
-          error = "Option #{sym} wasn't supplied."
-          log.log_error_message error
-          $stderr.puts "ERROR: #{error}"
-          exit -1
-        end
+        env_setting = ENV[option.upcase]
+        options[setting] ||= env_setting
       end
 
       options