nexpose_servicenow 0.4.15

data/lib/nexpose_servicenow/historical_data.rb

@@ -0,0 +1,234 @@
+require_relative './nexpose_helper'
+require_relative './nx_logger'
+
+module NexposeServiceNow
+  class HistoricalData
+    REPORT_FILE_NAME = "Nexpose-ServiceNow-latest_scans.csv"
+    STORED_FILE_NAME = "last_scan_data.csv"
+    TIMESTAMP_FILE_NAME = "last_vuln_run.csv"
+
+    def initialize(options)
+      @local_dir = File.expand_path(options[:output_dir])
+      @ids = options[:nexpose_ids]
+
+      @local_file = File.join(@local_dir, STORED_FILE_NAME)
+      @remote_file = File.join(@local_dir, REPORT_FILE_NAME)
+      @timestamp_file = File.join(@local_dir, TIMESTAMP_FILE_NAME)
+
+      @log = NexposeServiceNow::NxLogger.instance
+      @log.log_message "Retrieving environment variables."
+    end
+
+    #Filters the saved report down to the sites being queried
+    #This can then be used as a basis to update last_scan_data
+    def filter_report
+      #Create a full last_scan_data if it doesn't already exist
+      create_last_scan_data unless File.exist? @local_file
+
+      @log.log_message 'Filtering report down sites which will be queried'
+
+      remote_csv = load_scan_id_report
+      site_ids = @ids.map(&:to_s)
+      filtered_csv = remote_csv.delete_if { |r| !site_ids.include?(r['site_id']) }
+      File.open(@remote_file, 'w') do |f|
+        f.write(remote_csv.to_csv)
+      end
+
+      puts filtered_csv
+    end
+
+    #Reads the downloaded report containing LATEST scan IDs
+    def load_scan_id_report
+      @log.log_message "Loading scan data report"
+      unless File.exists? @remote_file
+        @log.log_message "No existing report file found."
+        return nil
+      end
+      CSV.read(@remote_file, headers: true)
+    end
+
+    #Loads the last scan data file as CSV.
+    #It may be necessary to create one first.
+    def load_last_scan_data
+      @log.log_message "Loading last scan data"
+
+      create_last_scan_data unless File.exist? @local_file
+      CSV.read(@local_file, headers: true)
+    end
+
+    def last_scan_ids(sites)
+      return [] if !File.exist? @local_file
+
+      csv = load_last_scan_data
+      last_scan_ids = {}
+      sites.each do |id|
+        row = csv.find { |r| r['site_id'] == id.to_s } || { 'last_scan_id' => '0' }
+        last_scan_ids[id.to_s] = row['last_scan_id']
+      end
+
+      last_scan_ids
+    end
+
+    #Compares stored scan IDs versus remote scan IDs.
+    #This determines which scans are included as filters.
+    def sites_to_scan
+      return @ids unless File.exist? @remote_file
+
+      @log.log_message 'Filtering for sites with new scans'
+
+      remote_csv = CSV.read(@remote_file, headers: true)
+      local_csv = load_last_scan_data
+
+      filtered_sites = []
+
+      @ids.each do |id|
+        remote_scan_id = remote_csv.find { |r| r['site_id'] == id.to_s } || {}
+        remote_scan_id = remote_scan_id['last_scan_id'] || 1
+
+        local_scan_id = local_csv.find { |r| r['site_id'] == id.to_s }
+        local_scan_id = local_scan_id['last_scan_id']
+
+        filtered_sites << id if local_scan_id.to_i < remote_scan_id.to_i
+      end
+
+      @ids = filtered_sites
+    end
+
+    #Creates a base last scan data file from a downloaded report
+    def create_last_scan_data
+      @log.log_message 'Creating base last scan data file'
+
+      csv = load_scan_id_report
+      csv.delete('finished')
+      csv.each { |l| l['last_scan_id'] = 0 }
+
+      save_last_scan_data(csv)
+    end
+
+    #Updates only the rows that were affected by this scan
+    def update_last_scan_data
+      @log.log_message "Updating last scan data"
+
+      if !(File.exist? @local_file) && !(File.exist? @remote_file) 
+        @log.log_message 'Last scan data does not exist yet.'
+        return
+      end
+
+      updated_csv = load_last_scan_data
+      remote_csv = load_scan_id_report
+
+      #merge changes in from remote_csv
+      remote_csv.each do |row|
+        updated_row = updated_csv.find { |r| r['site_id'] == row['site_id'] }
+        updated_row['last_scan_id'] = row['last_scan_id']
+      end
+
+      save_last_scan_data(updated_csv) 
+
+      #puts updated_csv
+    end
+
+    #Overwrite the last scan data file with new csv
+    def save_last_scan_data(csv)
+      @log.log_message 'Saving last scan data'
+      File.open(@local_file, 'w') do |f|
+        f.write(csv.to_csv)
+      end
+    end
+
+    def create_last_vuln_data(time=nil, sites=[])
+      @log.log_message 'Creating last vulnerability scan time file.'
+
+      time ||= Time.new(1985)
+      time = time.strftime("%Y-%m-%d") if time.class.to_s == 'Time'
+
+      file = File.expand_path(@timestamp_file)
+
+      CSV.open(file, 'w') do |csv|
+        csv << ['Last Scan Time', 'Sites']
+        csv << [time, sites.join(',')]
+      end
+    end
+
+    #Current IDs are inserted into the updated CSV file.
+    def last_vuln_run
+      @log.log_message 'Retrieving the last vulnerability timestamp'
+
+      create_last_vuln_data if !File.exist? @timestamp_file
+
+      file = File.expand_path(@timestamp_file)
+      csv = CSV.open(file, headers:true)
+      last_run = csv.readline['Last Scan Time']
+      csv.close
+
+      last_run
+    end
+
+    #Experimental
+    #These should probably return strings that can be mlog'd
+    def log_and_print(message)
+      puts message
+      @log.log_message message unless @log.nil?
+    end
+
+    def log_and_error(message)
+      $stderr.puts "ERROR: #{message}"
+      @log.log_message unless @log.nil?
+    end
+
+    def set_last_scan(nexpose_id, scan_id)
+      unless File.exist? @local_file
+        log_and_error 'Last scan data does not exist.'
+        log_and_error 'Can\'t set last scan data without existing file.'
+        exit -1
+      end
+
+      csv = load_last_scan_data
+      row = csv.find { |r| r['site_id'] == nexpose_id }
+      
+      if row.nil?
+        csv << [nexpose_id, scan_id]
+      else
+        row['last_scan_id'] = scan_id
+      end
+        
+      save_last_scan_data csv
+      
+      log_and_print 'Last scan data updated.'
+    end
+
+    def set_last_vuln(date, sites=nil)
+      create_last_vuln_data(date, sites)
+      log_and_print 'Last vuln data updated.'
+    end
+
+    def remove_local_file(filename)
+      unless File.exist? filename
+        log_and_error 'Can\'t remove file.'
+        log_and_error "File #{filename} cannot be located."
+        exit -1
+      end
+
+      new_name = "#{filename}.#{Time.new.strftime('%Y-%m-%d.%H:%M:%S')}"
+      begin
+        #Delete existing file with same name
+        File.delete new_name if File.exist? new_name
+        File.rename(filename, new_name)
+      rescue Exception => e
+        log_and_error "Error removing file:\n#{e}"
+        exit -1
+      end
+
+      log_and_print "File #{filename} removed"
+    end
+
+    def remove_last_scan_data
+      remove_local_file @local_file
+    end
+
+    def remove_last_vuln_data
+      remove_local_file @timestamp_file
+    end
+
+  end
+end

data/lib/nexpose_servicenow/nexpose_helper.rb

@@ -0,0 +1,162 @@
+require 'nexpose'
+require 'fileutils'
+require_relative './queries'
+require_relative './nx_logger'
+
+module NexposeServiceNow
+  class NexposeHelper
+    def initialize(url, port, username, password)
+      @log = NexposeServiceNow::NxLogger.instance
+    	@url = url
+    	@port = port
+    	@nsc = connect(username, password)
+
+      @timeout = 7200
+    end
+
+    def self.get_report_names(query_name, ids)
+      unless Queries.multiple_reports?(query_name)
+        return [ id: -1, report_name: get_report_name(query_name) ]
+      end
+
+      ids.map { |id| { id: id, report_name: get_report_name(query_name, id) }}
+    end
+
+    def self.get_report_name(query_name, id=nil)
+      name = "Nexpose-ServiceNow-#{query_name}"
+      name += "-#{id}" if Queries.multiple_reports?(query_name) && !id.nil?
+      name
+    end
+
+    def self.get_filepath(report_name, output_dir)
+      path = File.join output_dir, "#{report_name}.csv"
+      File.expand_path path
+    end
+
+    def create_report(query_name, ids, id_type, output_dir, query_options={})
+    	output_dir = File.expand_path(output_dir.to_s)
+
+      #A single report doesn't use site filters        
+      ids = [-1] unless Queries.multiple_reports?(query_name)
+
+      reports = []
+      ids.each do |id|
+        report_name = self.class.get_report_name(query_name, id)
+        clean_up_reports(report_name)
+
+        delta_options = create_query_options(query_options, id)
+
+        query = Queries.send(query_name, delta_options)
+        report_id = generate_config(query, report_name, [id], id_type)
+        run_report(report_id)
+        reports << save_report(report_name, report_id, output_dir)
+      end
+
+      return reports
+    end
+
+    def create_query_options(query_options, nexpose_id=nil)
+      options = {}
+      options[:vuln_query_date] = query_options[:vuln_query_date]
+
+      return options if nexpose_id == nil || nexpose_id == -1
+      return 0 if query_options[:last_scans].empty?
+      options[:last_scan_id] = "#{query_options[:last_scans][nexpose_id] || 0}"
+
+      @log.log_message("Query options: #{options}")
+      
+      options
+    end
+
+    def clean_up_reports(report_name)
+    	#log 'Deleting existing report...'
+      reports = @nsc.list_reports
+		  reports.select! { |r| r.name.start_with? report_name }
+		  reports.each do |r|
+		    @nsc.delete_report_config(r.config_id)
+		  end
+		end
+
+    def generate_config(query, report_name, ids, id_type)
+      @log.log_message "Generating query config with name #{report_name}..."
+    	report_config = Nexpose::ReportConfig.new(report_name, nil, 'sql')
+      report_config.add_filter('version', '2.0.1')
+      report_config.add_filter('query', query)
+
+      ids.each { |id| report_config.add_filter(id_type, id) if id != -1 }
+      report_id = report_config.save(@nsc, false)
+    end
+
+    def run_report(report_id)
+    	#log 'Running report...'
+      @nsc.generate_report(report_id, false)
+      wait_for_report(report_id)
+    end
+
+    def wait_for_report(id)
+      wait_until(:fail_on_exceptions => TRUE, :on_timeout => "Report generation timed out. Status: #{r = @nsc.last_report(id); r ? r.status : 'unknown'}") {
+        if %w(Failed Aborted Unknown).include?(@nsc.last_report(id).status)
+          raise "Report failed to generate! Status <#{@nsc.last_report(id).status}>"
+        end
+        @nsc.last_report(id).status == 'Generated'
+      }
+    end
+
+    def wait_until(options = {})
+      polling_interval = 15
+      time_limit = Time.now + @timeout
+      loop do
+        begin
+          val = yield
+          return val if val
+        rescue Exception => error
+          raise error if options[:fail_on_exceptions]
+        end
+        if Time.now >= time_limit
+          raise options[:on_timeout] if options[:on_timeout]
+          error ||= 'Timed out waiting for condition.'
+          raise error
+        end
+        sleep polling_interval
+      end
+    end
+
+    def save_report(report_name, report_id, output_dir)
+    	@log.log_message 'Saving report...'
+    	local_file_name = self.class.get_filepath(report_name, output_dir)
+		  File.delete(local_file_name) if File.exists? local_file_name
+
+		  #log 'Downloading report...'
+		  report_details = @nsc.last_report(report_id)
+		  File.open(local_file_name, 'wb') do |f| 
+        f.write(@nsc.download(report_details.uri))
+      end
+
+		  #Got the report, cleanup server-side
+		  @nsc.delete_report_config(report_id)
+      local_file_name
+    end
+
+  	def connect(username, password)
+  		begin
+		    console = Nexpose::Connection.new(@url, username, password)
+		    console.login
+		    @log.log_message 'Logged in.'
+		  rescue Exception => e
+		    @log.log_error_message 'Error logging in...'
+		    @log.log_error_message e
+
+        $stderr.puts "ERROR: Could not log in. Check log and Nexpose settings.\n#{e}"
+		    exit -1
+		  end
+
+      #@log.on_connect(@url, @port || 3780, console.session_id, "{}")
+		  console
+  	end
+
+    def all_sites
+      @nsc.sites.map { |s| s.id }
+    end
+
+  end
+end

data/lib/nexpose_servicenow/nx_logger.rb

@@ -0,0 +1,166 @@
+require 'fileutils'
+require 'json'
+require 'net/http'
+require 'singleton'
+
+module NexposeServiceNow
+  class NxLogger
+    include Singleton
+    LOG_PATH = "./logs/rapid7_%s.log"
+    KEY_FORMAT = "external.integration.%s"
+    PRODUCT_FORMAT = "%s_%s"
+
+    DEFAULT_LOG = 'integration'
+    PRODUCT_RANGE = 4..30
+    KEY_RANGE = 3..15
+
+    ENDPOINT = '/data/external/statistic/'
+
+    def initialize()
+      create_calls
+      @logger_file = get_log_path @product
+      setup_logging(true, 'info')
+    end
+
+    def setup_statistics_collection(vendor, product_name, gem_version)
+      begin
+        @statistic_key = get_statistic_key vendor
+        @product = get_product product_name, gem_version
+      rescue => e
+        #Continue
+      end
+    end
+
+    def setup_logging(enabled, log_level = 'info', stdout=false)
+      @stdout = stdout
+
+      log_message('Logging disabled.') unless enabled || @log.nil?
+      @enabled = enabled
+      return unless @enabled
+
+      @logger_file = get_log_path @product
+
+      require 'logger'
+      directory = File.dirname(@logger_file)
+      FileUtils.mkdir_p(directory) unless File.directory?(directory)
+      io = IO.for_fd(IO.sysopen(@logger_file, 'a'), 'a')
+      io.autoclose = false
+      io.sync = true
+      @log = Logger.new(io, 'weekly')
+      @log.level = if log_level.to_s.casecmp('info') == 0 
+                     Logger::INFO 
+                   else
+                     Logger::DEBUG
+                   end
+      log_message("Logging enabled at level <#{log_level}>")
+    end
+
+    def create_calls
+      levels = [:info, :debug, :error, :warn]
+      levels.each do |level|
+        method_name = 
+        define_singleton_method("log_#{level.to_s}_message") do |message|
+          puts message if @stdout
+          @log.send(level, message) unless !@enabled || @log.nil?
+        end
+      end
+    end
+
+    def log_message(message)
+      log_info_message message
+    end
+
+    def log_stat_message(message)
+    end
+
+    def get_log_path(product)
+      product.downcase! unless product.nil?
+      File.join(File.dirname(__FILE__), LOG_PATH % (product || DEFAULT_LOG))
+    end
+
+    def get_statistic_key(vendor)
+      if vendor.nil? || vendor.length < KEY_RANGE.min
+        log_stat_message("Vendor length is below minimum of <#{KEY_RANGE}>")
+        return nil
+      end
+
+      vendor.gsub!('-', '_')
+      vendor.slice! vendor.rindex('_') until vendor.count('_') <= 1
+
+      vendor.delete! "^A-Za-z0-9\_"
+
+      KEY_FORMAT % vendor[0...KEY_RANGE.max].downcase
+    end
+
+    def get_product(product, version)
+      return nil if ((product.nil? || product.empty?) || 
+                     (version.nil? || version.empty?))
+
+      product.gsub!('-', '_')
+      product.slice! product.rindex('_') until product.count('_') <= 1
+
+      product.delete! "^A-Za-z0-9\_"
+      version.delete! "^A-Za-z0-9\.\-"
+
+      product = (PRODUCT_FORMAT % [product, version])[0...PRODUCT_RANGE.max]
+
+      product.slice! product.rindex(/[A-Z0-9]/i)+1..-1
+
+      if product.length < PRODUCT_RANGE.min
+        log_stat_message("Product length below minimum <#{PRODUCT_RANGE.min}>.")
+        return nil
+      end
+      product.downcase
+    end
+
+    def generate_payload(statistic_value='')
+      product_name, separator, version = @product.to_s.rpartition('_')
+      payload_value = {'version' => version}.to_json
+
+      payload = {'statistic-key' => @statistic_key.to_s,
+                 'statistic-value' => payload_value,
+                 'product' => product_name}
+      JSON.generate(payload)
+    end
+
+    def send(nexpose_address, nexpose_port, session_id, payload)
+      header = {'Content-Type' => 'application/json',
+                'nexposeCCSessionID' => session_id,
+                'Cookie' => "nexposeCCSessionID=#{session_id}"}
+      req = Net::HTTP::Put.new(ENDPOINT, header)
+      req.body = payload
+      http_instance = Net::HTTP.new(nexpose_address, nexpose_port)
+      http_instance.use_ssl = true
+      http_instance.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      response = http_instance.start { |http| http.request(req) }
+      log_stat_message "Received code #{response.code} from Nexpose console."
+      log_stat_message "Received message #{response.msg} from Nexpose console."
+      log_stat_message 'Finished sending statistics data to Nexpose.'
+
+      response.code
+    end
+
+    def on_connect(nexpose_address, nexpose_port, session_id, value)
+      log_stat_message 'Sending statistics data to Nexpose'
+
+      if @product.nil? || @statistic_key.nil?
+        log_stat_message('Invalid product name and/or statistics key.')
+        log_stat_message('Statistics collection not enabled.')
+        return
+      end
+      
+      begin
+        payload = generate_payload value
+        send(nexpose_address, nexpose_port, session_id, payload)
+      rescue => e
+        #Let the program continue
+      end
+    end
+
+    #Used by net library for debugging 
+    def <<(value)
+      log_debug_message(value)
+    end
+
+  end
+end