nexpose_servicenow 0.6.2 → 0.7.1

data/lib/nexpose_servicenow/chunker.rb

@@ -63,7 +63,6 @@ module NexposeServiceNow
         else
           chunks << chunk
 
-          # TODO: Make generic?
           #Initialise chunk with this row information
           chunk = { site_id: site_id,
                     start: position, 

data/lib/nexpose_servicenow/csv_compare.rb

@@ -3,6 +3,23 @@ require 'tempfile'
 require 'csv-diff'
 require 'csv'
 
+
+class CSVDiff
+  class CSVSource
+    def find_field_indexes(key_fields, field_names)
+      key_fields.map do |field|
+        if field.is_a?(Integer)
+          field
+        else
+          field_names.index{ |field_name| field.to_s.downcase == field_name.downcase } or
+              raise ArgumentError, "Could not locate field '#{field}' in source field names: #{
+              field_names.join(', ')}"
+        end
+      end
+    end
+  end
+end
+
 module NexposeServiceNow
   class CsvCompare
     def self.get_columns(csv_file)

data/lib/nexpose_servicenow/helpers/connection_helper.rb

@@ -0,0 +1,79 @@
+require_relative '../queries/queries_base'
+
+module NexposeServiceNow
+  class ConnectionHelper
+    NOT_IMPL = 'Error: Method not implemented.'
+
+    def initialize(url, port, username, password, silo='')
+      @log = NexposeServiceNow::NxLogger.instance
+      @url = url
+      @port = port
+      @username = username
+      @password = password
+      @silo = silo
+
+      @timeout = 21600
+    end
+
+    def self.get_report_names(query_name, ids)
+      if QueriesBase.single_report?(query_name)
+        return [ id: -1, report_name: get_report_name(query_name) ]
+      end
+
+      ids.map do |id|
+        { id: id.first, report_name: get_report_name(query_name, id.first) }
+      end
+    end
+
+    def self.get_report_name(query_name, id=nil)
+      name = "Nexpose-ServiceNow-#{query_name}"
+      name += "-#{id}" unless QueriesBase.single_report?(query_name) || id.nil?
+      name
+    end
+
+    def create_query_options(query_options, nexpose_id=nil)
+      options = {}
+      options[:vuln_query_date] = query_options[:vuln_query_date]
+      options[:site_id] = nexpose_id
+      options[:id_type] = 'site'
+      options[:filters] = query_options[:filters] || {}
+
+      # Without a nexpose ID, we don't have a specific delta
+      return options if [nil, -1].include? nexpose_id
+
+      if query_options[:delta_values].empty?
+        error_msg = 'No delta values provided. Exiting...'
+        @log.log_error_message error_msg
+        raise error_msg
+      end
+
+      options[:delta] = "#{query_options[:delta_values][nexpose_id] || 0}"
+
+      @log.log_message("Query options: #{options}")
+
+      options
+    end
+
+    def self.get_filepath(report_name, output_dir)
+      path = File.join output_dir, "#{report_name}.csv"
+      File.expand_path path
+    end
+
+    def connect(username, password)
+      raise NOT_IMPL
+    end
+
+    def generate_report(query_name, ids, id_type, output_dir, query_options={})
+      raise NOT_IMPL
+    end
+
+    # Pulls the collection IDs from Nexpose (e.g. asset groups, sites)
+    def collection_ids(collection_type)
+      raise NOT_IMPL
+    end
+
+    def save_report(report_name, report_id, output_dir)
+      raise NOT_IMPL
+    end
+  end
+end

data/lib/nexpose_servicenow/helpers/data_warehouse_helper.rb

@@ -0,0 +1,134 @@
+require 'pg'
+require_relative './connection_helper'
+require_relative '../queries/warehouse_queries'
+
+module NexposeServiceNow
+  class DataWarehouseHelper < ConnectionHelper
+
+    SSL_MODE = 'prefer'
+    GRANULARITY = 500
+    CHUNK_LIMIT = 4_500_000
+
+    def initialize(url, port, username, password, silo='')
+      super(url, port, username, password, silo)
+    end
+
+    def connect
+      @log.log_message 'Creating DWH connection'
+      PG::Connection.open(:host => @url,
+                          :dbname => @silo,
+                          :port => @port,
+                          :user => @username,
+                          :password => @password,
+                          :sslmode => SSL_MODE)
+    end
+
+    def generate_report(query_name, ids, id_type, output_dir, query_options={})
+      output_dir = File.expand_path(output_dir.to_s)
+
+      #A single report doesn't use site filters
+      ids = [-1] if WarehouseQueries.single_report?(query_name)
+
+      page_size = query_options[:page_size]
+      row_limit = query_options[:row_limit]
+
+      chunks = []
+
+      base_name = "query_#{query_name}"
+
+      ids.each do |id|
+        delta_options = create_query_options(query_options, id)
+        query = WarehouseQueries.send(query_name, delta_options)
+
+        # Open the CSV file to write as pages are retrieved
+        report_name = self.class.get_report_name(query_name, id)
+        @log.log_message "Running query for #{report_name}"
+
+        local_file_name = self.class.get_filepath(report_name, output_dir)
+        csvFile = File.open(local_file_name, 'wb')
+
+        conn = connect
+        conn.transaction do
+          table_name = "query_#{query_name}"
+          table_name = "#{base_name}_#{id}" if id && id.to_i > 0
+
+          @log.log_message "Creating cursor: #{table_name}"
+          conn.exec("DECLARE #{table_name} CURSOR FOR #{query}")
+          res = conn.exec("FETCH #{page_size} FROM #{table_name}")
+
+          # Headers
+          headers = res.fields.join(',')
+          csvFile.puts(headers)
+
+          # Declare the initial chunk
+          chunk = { start: csvFile.pos, length: 0, row_count: 0 }
+          chunk[:site_id] = id unless id.nil? || id == -1
+
+          # Should we overwrite 'res' to release the memory?
+          all_lines = res.values.map { |r| r.join(',') }
+
+          # Table declared, so keep reading pages until it's consumed
+          data_left = true
+          while(data_left)
+            # Lift out a number of lines in a chunk
+            text = all_lines.slice!(0, GRANULARITY)
+
+            # Try to get the next page
+            if text.nil? || text.count == 0
+              res = conn.exec("FETCH #{page_size} FROM #{table_name}")
+              if res.values.count == 0
+                chunks << chunk
+                break
+              end
+              all_lines = res.values.map { |r| r.join(',') }
+              next
+            end
+
+            # Work out the details for this chunk
+            line_count = text.count
+            text = text.join("\n")
+            text << "\n"
+            byte_size = text.bytesize
+
+            # Test whether limits would be exceeded
+            below_row_limit = chunk[:row_count] + line_count <= row_limit
+            below_size_limit = chunk[:length] + byte_size < CHUNK_LIMIT
+
+            if below_size_limit && below_row_limit
+              chunk[:length] += byte_size
+              chunk[:row_count] += line_count
+            else
+              chunks << chunk.dup
+              # Store the current pos since the next chunk isn't written
+              chunk[:start] = csvFile.pos
+              chunk[:length] = byte_size
+              chunk[:row_count] = line_count
+            end
+
+            csvFile.write(text)
+          end
+        end
+
+        conn.finish
+
+        # Close the file for this specific report
+        csvFile.close
+        @log.log_message "Report generated: #{report_name}"
+      end
+
+      chunks.each { |c| c.delete :row_count }
+
+      @log.log_message "Finished running query: #{query_name}"
+      chunks.to_json
+    end
+
+    def collection_ids
+      @log.log_message 'Retrieving list of site IDs'
+      connection = connect
+      query = 'select site_id from dim_site'
+      result = connection.query(query)
+      result.map { |r| r['site_id'] }
+      connection.finish
+    end
+  end
+end

data/lib/nexpose_servicenow/{nexpose_helper.rb → helpers/nexpose_console_helper.rb}

@@ -1,108 +1,50 @@
 require 'nexpose'
 require 'fileutils'
-require_relative './queries'
-require_relative './nx_logger'
-require_relative './csv_compare'
+require_relative './connection_helper'
+require_relative '../queries/nexpose_queries'
+require_relative '../nx_logger'
+require_relative '../csv_compare'
 
 module NexposeServiceNow
-  class NexposeHelper
-    def initialize(url, port, username, password)
-      @log = NexposeServiceNow::NxLogger.instance
-      @url = url
-      @port = port
-      @username = username
-      @password = password
+  class NexposeConsoleHelper < ConnectionHelper
+    def initialize(url, port, username, password, silo='')
+      super(url, port, username, password, silo)
 
-      @nsc = connect(username, password)
-
-      @timeout = 21600
-    end
-
-    def self.get_report_names(query_name, ids)
-      unless Queries.multiple_reports?(query_name)
-        return [ id: -1, report_name: get_report_name(query_name) ]
-      end
-
-      ids.map { |id| { id: id, report_name: get_report_name(query_name, id) } }
-    end
-
-    def self.get_report_name(query_name, id=nil)
-      name = "Nexpose-ServiceNow-#{query_name}"
-      name += "-#{id}" if Queries.multiple_reports?(query_name) && !id.nil?
-      name
-    end
-
-    def self.get_filepath(report_name, output_dir)
-      path = File.join output_dir, "#{report_name}.csv"
-      File.expand_path path
+      @nsc = connect
     end
 
-    def create_report(query_name, ids, id_type, output_dir, query_options={})
+    def generate_report(query_name, ids, id_type, output_dir, query_options={})
       output_dir = File.expand_path(output_dir.to_s)
 
       #A single report doesn't use site filters        
-      ids = [-1] unless Queries.multiple_reports?(query_name)
+      ids = [-1] if NexposeQueries.single_report?(query_name)
 
       #If running latest_scans, send up information
       if query_name == 'latest_scans'
         @log.on_connect(@url, @port, @nsc.session_id, '{}')
       end
 
-      reports = []
       ids.each do |id|
         report_name = self.class.get_report_name(query_name, id)
         clean_up_reports(report_name)
 
-        # Should use the value from historical_data.rb
-        default_delta = if id_type.to_s == 'asset_group'
-                          '1985-01-01 00:00:00'
-                        elsif id_type.to_s == 'site'
-                          0
-                        end
+        delta_options = create_query_options(query_options, id)
 
-        delta_options = create_query_options(query_options, id, default_delta)
+        query = NexposeQueries.send(query_name, delta_options)
 
-        # TODO: Refactor this into 'create_query_options'
-        delta_options[:site_id] = id
-        delta_options[:id_type] = id_type.to_s
-        delta_options[:filters] = query_options[:filters] || {}
+        report_id = generate_config(query, report_name, [id], id_type)
 
-        min_cvss = if delta_options[:filters][:cvss].nil?
-                     0
-                   else
-                     delta_options[:filters][:cvss].first
-                   end
-
-        query = Queries.send(query_name, delta_options)
-
-        report_id = generate_config(query, report_name, [id], id_type, min_cvss)
-
-        run_report(report_id)
+        run_report(report_id, report_name)
         local_report_name = save_report(report_name, report_id, output_dir)
-        reports << local_report_name
 
-        if Queries.csv_diff_required?(query_name)
+        if NexposeQueries.csv_diff_required?(query_name)
           @log.log_message "Calculating diff for #{local_report_name}..."
           CsvCompare.update_report_with_diff(local_report_name,
-                                             Queries.query_keys(query_name))
+                                             NexposeQueries.query_keys(query_name))
         end
       end
 
-      reports
-    end
-
-    def create_query_options(query_options, nexpose_id=nil, default=0)
-      options = {}
-      options[:vuln_query_date] = query_options[:vuln_query_date]
-
-      return options if nexpose_id.nil? || nexpose_id == -1
-      return 0 if query_options[:delta_values].empty?
-
-      options[:delta] = "#{query_options[:delta_values][nexpose_id] || default}"
-
-      @log.log_message("Query options: #{options}")
-      
-      options
+      nil
     end
 
     def clean_up_reports(report_name)
@@ -111,8 +53,8 @@ module NexposeServiceNow
       reports.each { |r| @nsc.delete_report_config(r.config_id) }
     end
 
-    def generate_config(query, report_name, ids, id_type, min_severity=0)
-      @nsc = connect(@username, @password)
+    def generate_config(query, report_name, ids, id_type)
+      @nsc = connect
       @log.log_message "Generating report config with name #{report_name}..."
       report_config = Nexpose::ReportConfig.new(report_name, nil, 'sql')
       report_config.add_filter('version', '2.0.1')
@@ -129,8 +71,8 @@ module NexposeServiceNow
       report_id
     end
 
-    def run_report(report_id)
-      @log.log_message "Running report #{report_id}..."
+    def run_report(report_id, report_name)
+      @log.log_message "Running Report #{report_name}, ID: #{report_id}..."
       @nsc.generate_report(report_id, false)
       wait_for_report(report_id)
     end
@@ -152,11 +94,16 @@ module NexposeServiceNow
           val = yield
           return val if val
         rescue Exception => error
+          @log.log_error_message("Error during wait_until: #{error}")
           raise error if options[:fail_on_exceptions]
         end
         if Time.now >= time_limit
-          raise options[:on_timeout] if options[:on_timeout]
+          if options[:on_timeout]
+            @log.log_error_message("#{options[:on_timeout]}. Exiting...")
+            raise options[:on_timeout]
+          end
           error ||= 'Timed out waiting for condition.'
+          @log.log_error_message("#{error}. Exiting...")
           raise error
         end
         sleep polling_interval
@@ -164,7 +111,7 @@ module NexposeServiceNow
     end
 
     def save_report(report_name, report_id, output_dir)
-      @log.log_message 'Saving report...'
+      @log.log_message "Saving report - Name: #{report_name}, ID: #{report_id}..."
       local_file_name = self.class.get_filepath(report_name, output_dir)
       File.delete(local_file_name) if File.exists? local_file_name
 
@@ -176,9 +123,9 @@ module NexposeServiceNow
       
       begin
         # Refresh the connection
-        @nsc = connect(@username, @password)
+        @nsc = connect
 
-        #Got the report, cleanup server-side
+        # Got the report, cleanup server-side
         @nsc.delete_report_config(report_id)
       rescue
         @log.log_error_message 'Error deleting report'
@@ -187,9 +134,9 @@ module NexposeServiceNow
       local_file_name
     end
 
-    def connect(username, password)
+    def connect
       begin
-        connection = Nexpose::Connection.new(@url, username, password, @port)
+        connection = Nexpose::Connection.new(@url, @username, @password, @port)
         connection.login
         @log.log_message 'Logged in.'
       rescue Exception => e

data/lib/nexpose_servicenow/historical_data.rb

@@ -1,325 +1,72 @@
-require_relative './nexpose_helper'
 require_relative './nx_logger'
 
 module NexposeServiceNow
   class HistoricalData
 
-    REPORT_FILE = 'Nexpose-ServiceNow-latest_scans.csv'
-    STORED_FILE = 'last_scan_data.csv'
-    TIMESTAMP_FILE = 'last_vuln_run.csv'
-    NEW_TIMESTAMP_FILE = 'new_vuln_timestamp.csv'
-
-    DAG_TIMESTAMP_FILE = 'last_scan_data_dag.csv'
-    NEW_DAG_TIMESTAMP_FILE = 'new_dag_timestamp.csv'
-
-    ASSET_GROUP_FILE = 'Nexpose-ServiceNow-asset_groups-old.csv'
-    DIFFERENTIAL_FILE_REGEX = 'Nexpose-ServiceNow-*-old.csv'
-
     SITE_IDENTIFIER = 'site_id'
     SITE_DELTA_VALUE = 'last_scan_id'
-    SITE_BASE_VALUE = 0
+    SITE_TIMESTAMP_VALUE = 'finished'
+
+    DEFAULT_TIMESTAMP_VALUE = '1985-01-01 12:00:00'
 
-    DAG_IDENTIFIER = 'asset_group_id'
-    DAG_DELTA_VALUE = 'last_import'
-    DAG_BASE_VALUE = '1985-01-01 00:00:00'
+    GROUP_FILE_PATTERN = 'Nexpose-ServiceNow-asset_group*.csv'
 
-    def initialize(output_dir, nexpose_ids, id_type, start_time)
+    def initialize(output_dir)
       local_dir = File.expand_path(output_dir)
-      @ids = nexpose_ids
-      @id_type = id_type
+      @remote_file = File.join(local_dir, 'Nexpose-ServiceNow-latest_scans.csv')
+      @log = NexposeServiceNow::NxLogger.instance
+    end
 
-      if @id_type == :site
-        current_file = STORED_FILE
-        new_file = REPORT_FILE
-      elsif @id_type == :asset_group
-        current_file = DAG_TIMESTAMP_FILE
-        new_file = NEW_DAG_TIMESTAMP_FILE
-      end
+    # Filters out irrelevant site IDs
+    def filter_ids(site_ids)
+      original_site_ids = site_ids.keys
+      default_values = ['0', "\"#{DEFAULT_TIMESTAMP_VALUE}\""]
 
-      @start_time = start_time
+      # Reject if the delta value matches a default value
+      site_ids.reject! do |site, delta|
+        default_values.any? { |v| delta == v }
+      end
 
-      @local_file = File.join(local_dir, current_file)
-      @remote_file = File.join(local_dir, new_file)
+      # Get a list of excluded sites
+      excluded = original_site_ids - site_ids.keys
 
-      # File containing the timestamp used in vulnerability queries
-      @timestamp_file = File.join(local_dir, TIMESTAMP_FILE)
-      @prev_timestamp_file = File.join(local_dir, NEW_TIMESTAMP_FILE)
+      if excluded.count > 0
+        @log.log_message("Excluding #{excluded} from old vulns query.")
+      end
 
-      @log = NexposeServiceNow::NxLogger.instance
+      site_ids
     end
 
-    # TODO: Remove site references here? Will there be remote CSV here if we're using DAGs? No scan IDs.
-    # Filters the saved report down to the sites being queried
-    # This can then be used as a basis to update last_scan_data
-    def filter_report
-      # Create a full last_scan_data if it doesn't already exist
-      create_base_delta_file unless File.exist? @local_file
-
+    # Replaces scan IDs of sites that aren't to be imported with 0
+    def filter_report(site_ids=[])
       @log.log_message 'Filtering report down to sites which will be queried'
 
       remote_csv = load_scan_id_report
-      nexpose_ids = @ids.map(&:to_s)
-      identifier = if @id_type == :site
-                     SITE_IDENTIFIER
-                   elsif @id_type ==:asset_group
-                     DAG_IDENTIFIER
-                   end
+      nexpose_ids = site_ids.map(&:to_s)
 
-      if @id_type == :asset_group
-        header = [DAG_IDENTIFIER, DAG_DELTA_VALUE]
-        rows = []
+      return remote_csv if nexpose_ids.first == '0'
 
-        @ids.each do |i|
-          rows << CSV::Row.new(header, [i, @start_time])
+      remote_csv.each do |row|
+        unless nexpose_ids.include?(row[SITE_IDENTIFIER])
+          row[SITE_DELTA_VALUE] = '0'
+          row[SITE_TIMESTAMP_VALUE] = DEFAULT_TIMESTAMP_VALUE
         end
-
-        remote_csv = CSV::Table.new(rows)
-      end
-
-      # TODO: Why is this done? Aren't these already filtered?
-      filtered_csv = remote_csv.delete_if do |r|
-        !nexpose_ids.include?(r[identifier])
-      end
-
-      File.open(@remote_file, 'w') do |f|
-        f.write(remote_csv.to_csv)
       end
 
-      puts filtered_csv
+      remote_csv
     end
 
     # Reads the downloaded report containing LATEST scan IDs
     def load_scan_id_report
       @log.log_message 'Loading scan data report'
       unless File.exists? @remote_file
-        @log.log_message 'No existing report file found.'
-        return nil
+        error = 'Latest scan report file could not be found.'
+        @log.log_message error
+        raise error
       end
       CSV.read(@remote_file, headers: true)
     end
 
-    # Loads the last scan data file as CSV.
-    # It may be necessary to create one first.
-    def load_last_scan_data
-      @log.log_message 'Loading last scan data.'
-
-      create_base_delta_file unless File.exist? @local_file
-      CSV.read(@local_file, headers: true)
-    end
-
-    def stored_delta_values(nexpose_ids)
-      return [] if !File.exist? @local_file
-
-      if @id_type == :site
-        identifier = SITE_IDENTIFIER
-        delta_column = SITE_DELTA_VALUE
-        base_value = SITE_BASE_VALUE
-      elsif @id_type == :asset_group
-        identifier = DAG_IDENTIFIER
-        delta_column = DAG_DELTA_VALUE
-        base_value = DAG_BASE_VALUE
-      end
-
-      csv = load_last_scan_data
-      delta_values = {}
-      nexpose_ids.each do |id|
-        row = csv.find { |r| r[identifier] == id.to_s }
-        row ||= { delta_column => base_value }
-        delta_values[id.to_s] = row[delta_column]
-      end
-
-      delta_values
-    end
-
-    # Compares stored scan IDs versus remote scan IDs.
-    # This determines which scans are included as filters.
-    def collections_to_import(previously_imported_only=false)
-      return @ids unless File.exist? @remote_file
-      @log.log_message "Filtering for #{@id_type}s with new scans"
-      self.send("#{@id_type}s_to_import", previously_imported_only)
-    end
-
-    def sites_to_import(previously_imported_only=false)
-      remote_csv = CSV.read(@remote_file, headers: true)
-      local_csv = load_last_scan_data
-
-      filtered_sites = []
-
-      @ids.each do |id|
-        remote_scan_id = remote_csv.find { |r| r['site_id'] == id.to_s } || {}
-        remote_scan_id = remote_scan_id['last_scan_id'] || 1
-
-        local_scan_id = local_csv.find { |r| r['site_id'] == id.to_s } || {}
-        local_scan_id = local_scan_id['last_scan_id'] || 0
-
-        # Check if only allowing sites which were previously imported
-        next if local_scan_id.to_s == '0' && previously_imported_only
-
-        filtered_sites << id if local_scan_id.to_i < remote_scan_id.to_i
-      end
-
-      @ids = filtered_sites
-    end
-
-    def asset_groups_to_import(previously_imported_only=false)
-      filtered_asset_groups = []
-      local_csv = load_last_scan_data
-
-      @ids.each do |id|
-        local_id = local_csv.find { |r| r[DAG_IDENTIFIER] == id.to_s } || {}
-        local_id = local_id[DAG_DELTA_VALUE] || DAG_BASE_VALUE
-
-        next if local_id == DAG_BASE_VALUE && previously_imported_only
-
-        filtered_asset_groups << id
-      end
-
-      @ids = filtered_asset_groups
-    end
-
-    # Creates a base last scan data file from a downloaded report
-    def create_base_delta_file
-      @log.log_message 'Creating base delta file'
-      self.send("create_#{@id_type}_base_file")
-    end
-
-    def create_site_base_file
-      csv = load_scan_id_report
-      csv.delete('finished')
-      csv.each { |l| l['last_scan_id'] = SITE_BASE_VALUE }
-
-      save_last_scan_data(csv)
-    end
-
-    def create_asset_group_base_file
-      CSV.open(@local_file, 'w') do |csv|
-        csv << %w(asset_group_id last_import)
-        @ids.each do |n|
-          csv << [n, DAG_BASE_VALUE]
-        end
-      end
-    end
-
-    # Updates only the rows that were affected by this scan
-    def update_delta_file
-      @log.log_message 'Updating last scan data'
-
-      if !(File.exist? @local_file) && !(File.exist? @remote_file)
-        @log.log_message 'Last scan data does not exist yet.'
-        return
-      end
-
-      updated_csv = load_last_scan_data
-      remote_csv = load_scan_id_report
-
-      method = "update_#{@id_type}_delta_file"
-      updated_csv = self.send(method, updated_csv, remote_csv)
-
-      save_last_scan_data(updated_csv)
-    end
-
-    def update_site_delta_file(updated_csv, remote_csv)
-      #merge changes in from remote_csv
-      remote_csv.each do |row|
-        updated_row = updated_csv.find { |r| r['site_id'] == row['site_id'] }
-        if updated_row.nil?
-          row.delete 'finished'
-          updated_csv << row
-        else
-          updated_row['last_scan_id'] = row['last_scan_id']
-        end
-      end
-
-      updated_csv
-    end
-
-    def update_asset_group_delta_file(updated_csv, remote_csv)
-      #merge changes in from remote_csv
-      remote_csv.each do |row|
-        updated_row = updated_csv.find do |r|
-          r['asset_group_id'] == row['asset_group_id']
-        end
-
-        if updated_row.nil?
-          updated_csv << row
-        else
-          updated_row['last_import'] = row['last_import']
-        end
-      end
-
-      # Insert any NEW IDs with baseline time
-      @ids.each do |i|
-        row = updated_csv.find { |r| r[DAG_IDENTIFIER] == i }
-        updated_csv << [i, DAG_BASE_VALUE] if row.nil?
-      end
-
-      updated_csv
-    end
-
-    # Overwrite the last scan data file with new csv
-    def save_last_scan_data(csv)
-      @log.log_message 'Saving last scan data'
-      File.open(@local_file, 'w') do |f|
-        f.write(csv.to_csv)
-      end
-    end
-
-    def save_vuln_timestamp(nexpose_ids=[])
-      start_time = Time.new
-
-      #Read timestamp from new timestamp file (substitute base time)
-      if File.exist? @prev_timestamp_file
-        file = File.expand_path(@prev_timestamp_file)
-        csv = CSV.open(file, headers:true)
-        line = csv.readline
-        last_run = line['Last Scan Time']
-        last_sites = line['Sites']
-        csv.close
-
-        File.delete(file)
-      end
-
-      last_run ||= Time.new(1985)
-      last_sites ||= []
-      last_run = last_run.strftime('%Y-%m-%d') if last_run.class.to_s == 'Time'
-      create_last_vuln_data(last_run, last_sites)
-
-      file = File.expand_path(@prev_timestamp_file)
-      CSV.open(file, 'w') do |csv|
-        csv << ['Last Scan Time', 'Sites']
-        csv << [start_time.strftime('%Y-%m-%d'), nexpose_ids.join(',')]
-      end
-    end
-
-    def create_last_vuln_data(time=nil, nexpose_ids=[])
-      @log.log_message 'Creating last vulnerability scan time file.'
-
-      time ||= Time.new(1985)
-      time = time.strftime('%Y-%m-%d') if time.class.to_s == 'Time'
-      nexpose_ids = nexpose_ids.join(',') if nexpose_ids.class.to_s == 'Array'
-
-      file = File.expand_path(@timestamp_file)
-
-      CSV.open(file, 'w') do |csv|
-        csv << ['Last Scan Time', 'Sites']
-        csv << [time, nexpose_ids]
-      end
-    end
-
-    # Current IDs are inserted into the updated CSV file.
-    def last_vuln_run
-      @log.log_message 'Retrieving the last vulnerability timestamp'
-
-      create_last_vuln_data if !File.exist? @timestamp_file
-
-      file = File.expand_path(@timestamp_file)
-      csv = CSV.open(file, headers:true)
-      last_run = csv.readline['Last Scan Time']
-      csv.close
-
-      last_run
-    end
-
     #########################################################
     #                    Experimental                       #
     #########################################################
@@ -335,77 +82,21 @@ module NexposeServiceNow
       @log.log_error_message message unless @log.nil?
     end
 
-    def set_last_scan(nexpose_id, scan_id)
-      unless File.exist? @local_file
-        log_and_error 'Last scan data does not exist.'
-        log_and_error 'Can\'t set last scan data without existing file.'
-        exit -1
-      end
-
-      csv = load_last_scan_data
-      row = csv.find { |r| r['site_id'] == nexpose_id }
-
-      if row.nil?
-        csv << [nexpose_id, scan_id]
-      else
-        row['last_scan_id'] = scan_id
-      end
-
-      save_last_scan_data csv
-
-      log_and_print 'Last scan data updated.'
-    end
-
-    def set_last_vuln(date, sites=nil)
-      create_last_vuln_data(date, sites)
-      log_and_print 'Last vuln data updated.'
-    end
-
-    def remove_local_file(filename, action)
-      unless File.exist? filename
-        log_and_error 'Can\'t remove file.'
-        log_and_error "File #{filename} cannot be located."
-        return
-      end
-
-      begin
-        self.send("by_#{action}", filename)
-      rescue Exception => e
-        log_and_error "Error removing file:\n#{e}"
-        return
-      end
-
-      log_and_print "File #{filename} removed"
-    end
-
-    def by_rename(filename)
-      new_name = "#{filename}.#{Time.new.strftime('%Y-%m-%d_%H-%M-%S')}"
-      File.delete new_name if File.exist? new_name
-      File.rename(filename, new_name)
-    end
-
-    def by_delete(filename)
-      File.delete filename
-    end
-
-    def remove_last_scan_data
-      remove_local_file @local_file, 'rename'
-      remove_local_file @remote_file, 'delete'
-    end
-
-    def remove_last_vuln_data
-      remove_local_file @timestamp_file, 'rename'
-      remove_local_file @prev_timestamp_file, 'delete'
-    end
-
-    def remove_last_diff_comparison_data(output_dir)
+    # Deletes all of the CSV files matching the pattern
+    def remove_diff_files(output_dir)
       local_path = File.expand_path(output_dir)
-      remove_local_file (local_path + ASSET_GROUP_FILE), 'rename'
-
-      Dir[local_path + DIFFERENTIAL_FILE_REGEX].each do |file|
-        remove_local_file file, 'delete'
+      group_csv_files = Dir.glob(File.join(local_path, GROUP_FILE_PATTERN))
+
+      group_csv_files.each do |file|
+        begin
+          File.delete file
+          log_and_print "File #{file} deleted."
+        rescue Exception => e
+          log_and_error "Error removing file:\n#{e}"
+        end
       end
     end
   end
 end
 
+