nexpose_servicenow 0.7.3 → 0.8.0

data/lib/nexpose_servicenow/chunker.rb

@@ -1,109 +0,0 @@
-#TODO: Check if using site_id is OK. If the object is changed, the code on SN will likely need changed.
-
-module NexposeServiceNow
-  class Chunker
-    def initialize(report_details, row_limit)
-      @row_limit = row_limit
-      @size_limit = 4_500_000
-      @report_details = report_details
-      @header = get_header
-
-      setup_logging
-    end
-
-    def setup_logging
-      @log = NexposeServiceNow::NxLogger.instance
-      @log.log_message("Chunk File Limit:\t#{@size_limit}MB")
-      @log.log_message("Chunk Row Limit:\t#{@row_limit}")
-    end
-
-    # Grab the header from the first file
-    def get_header
-      file = File.open(@report_details.first[:report_name], 'r')
-      header = file.readline
-      file.close
-
-      header
-    end
-
-    def preprocess
-      all_chunks = []
-      @report_details.each do |report|
-        @log.log_message("Dividing file #{report[:report_name]} into chunks.")
-        chunks = process_file(report[:report_name], report[:id])
-        all_chunks.concat chunks
-      end 
-
-      @log.log_message("Files divided into #{all_chunks.count} chunks")
-
-      puts all_chunks.to_json
-    end
-
-    def process_file(file_path, site_id=nil)
-      relative_size_limit = @size_limit - @header.bytesize
-      chunk = { site_id: site_id,
-                start: @header.bytesize,
-                length: 0, 
-                row_count: 0 }
-      
-      chunks = []
-      csv_file = CSV.open(file_path, 'r', headers: true)
-      while(true)
-        position = csv_file.pos
-        line = csv_file.shift
-        row_length = line.to_s.bytesize
-        
-        if line.nil?
-          chunks << chunk
-          break
-        elsif chunk[:length]+row_length < relative_size_limit &&
-              chunk[:row_count] + 1 < @row_limit
-          chunk[:length] += row_length
-          chunk[:row_count] += 1
-        else
-          chunks << chunk
-
-          #Initialise chunk with this row information
-          chunk = { site_id: site_id,
-                    start: position, 
-                    length: row_length, 
-                    row_count: 1 }
-        end
-      end
-      csv_file.close
-
-      #Should we include the row count?
-      chunks.each do |c| 
-        c.delete :row_count
-
-        #Should we do this...?
-        c.delete :site_id if c[:site_id].nil? || c[:site_id] == -1
-      end
-
-      chunks
-    end
-
-    def get_file(site_id=nil)
-      # -1 indicates a single query report
-      return @report_details.first[:report_name] if site_id.to_i <= 0
-
-      report = @report_details.find { |r| r[:id].to_s == site_id.to_s }
-      report[:report_name]
-    end
-
-    def read_chunk(start, length, site_id=nil)
-      file_path = get_file(site_id)
-      msg = "Returning chunk. Start: #{start}, " \
-            "Length: #{length}, File: #{file_path}"
-      @log.log_message(msg)
-
-      #If the header isn't in the chunk, prepend it
-      header = start == 0 ? '' : @header
-
-      file = File.open(file_path, 'rb')
-      file.seek(start)
-      puts header + file.read(length)
-      file.close
-    end
-  end
-end

data/lib/nexpose_servicenow/csv_compare.rb

@@ -1,177 +0,0 @@
-require 'fileutils'
-require 'tempfile'
-require 'csv-diff'
-require 'csv'
-
-
-class CSVDiff
-  class CSVSource
-    def find_field_indexes(key_fields, field_names)
-      key_fields.map do |field|
-        if field.is_a?(Integer)
-          field
-        else
-          field_names.index{ |field_name| field.to_s.downcase == field_name.downcase } or
-              raise ArgumentError, "Could not locate field '#{field}' in source field names: #{
-              field_names.join(', ')}"
-        end
-      end
-    end
-  end
-end
-
-module NexposeServiceNow
-  class CsvCompare
-    def self.get_columns(csv_file)
-      columns = ''
-      File::open(csv_file,'r') do |f|
-        columns = f.readline.rstrip
-      end
-
-      columns.split(',')
-    end
-
-    def self.get_row(value, columns, status)
-      columns.map { |c| value.fields[c] }.push(status)
-    end
-
-    def self.get_delete(value, columns)
-      self.get_row(value, columns, 'old')
-    end
-
-    def self.get_add(value, columns)
-      self.get_row(value, columns, 'new')
-    end
-
-    def self.update_to_old(value, columns)
-      self.update_to_row(value, columns, 0, 'old')
-    end
-
-    def self.update_to_new(value, columns)
-      self.update_to_row(value, columns, 1, 'new')
-    end
-
-    def self.update_to_row(value, columns, index, status)
-      row = []
-      columns.each do |c|
-        val = value.fields[c]
-        row << (val.kind_of?(Array) ? val[index] : val)
-      end
-      row.push(status)
-    end
-
-    def self.append_to_filename(current_filename, string_to_append)
-      extension = File.extname current_filename
-      name = File.basename current_filename, extension
-      path = File.dirname current_filename
-
-      "#{path}/#{name}-#{string_to_append}#{extension}"
-    end
-
-    def self.update_report_with_diff(report_file, key_fields=[0])
-      old_filename = self.append_to_filename(report_file, 'old')
-      new_filename = self.append_to_filename(report_file, 'new')
-
-      # Report is 'new' file for purpose of diff
-      FileUtils.mv(report_file, new_filename)
-
-      # If the old file doesn't exist, we can just add the status column
-      if File.exists?(old_filename)
-        self.create_csv_diff(old_filename,
-                             new_filename,
-                             report_file,
-                             key_fields)
-      else
-        self.overwrite_existing_report(new_filename, report_file)
-      end
-
-      # 'new' file becomes the basis of comparison next time
-      FileUtils.mv(new_filename, old_filename, :force => true)
-    end
-
-    # Instead of diffing, append 'status' column to an existing file
-    def self.overwrite_existing_report(source_file, target_file)
-      temp = Tempfile.new("#{File.basename source_file}tmp")
-
-      #TODO: Don't do the column_written check every time
-
-      CSV.open(temp, 'w') do |temp_csv|
-        column_written = false
-        new_column_value = ['status']
-        CSV.foreach(source_file) do |orig|
-          temp_csv << (orig + new_column_value)
-
-          unless column_written
-            new_column_value = ['new']
-            column_written = true
-          end
-        end
-      end
-
-      temp.close
-      FileUtils.mv(temp, target_file, :force => true)
-    end
-
-    def self.create_csv_diff(old_file, new_file, target_file, key_fields=[0])
-      old_temp = Tempfile.new("#{File.basename old_file}tmp")
-      new_temp = Tempfile.new("#{File.basename new_file}tmp")
-
-      File.open(old_file) do |f|
-        IO.copy_stream(f, old_temp)
-      end
-      File.open(new_file) do |f|
-        IO.copy_stream(f, new_temp)
-      end
-
-      begin
-        diff = CSVDiff.new(
-          File.expand_path(old_temp),
-          File.expand_path(new_temp),
-          ignore_moves: true,
-          key_fields: key_fields
-        )
-      rescue Exception => e
-        file_name = File.basename target_file
-        raise "Unable to diff file: #{file_name}. \nError received: #{e}"
-      ensure
-        old_temp.close!
-        new_temp.close!
-      end
-
-      columns = self.get_columns(new_file)
-
-      CSV.open(target_file, 'wb') do |csv|
-        csv << (columns+['status'])
-
-        diff.deletes.each_value { |v| csv << get_delete(v, columns) }
-        diff.adds.each_value { |v| csv << get_add(v, columns) }
-
-        if key_fields.count == 1
-          # If only a single key field, we don't need the old values
-          # Just grab the row and let ServiceNow coalesce and update the row
-          update_rows = diff.updates.each_value.map { |u| u.row }
-          update_rows = update_rows.map(&:to_i).sort
-          update_row_num = update_rows.shift
-          current_row_num = 0
-
-          CSV.foreach(new_file, headers: true) do |new_csv|
-            current_row_num = current_row_num + 1
-            next unless current_row_num == update_row_num
-
-            csv << new_csv.push('new')
-            update_row_num = update_rows.shift
-            break if update_row_num == nil
-          end
-        elsif key_fields.count == 2
-          # Multiple key fields result in row "updates"
-          diff.updates.each_value do |v|
-            csv << self.update_to_old(v, columns)
-            csv << self.update_to_new(v, columns)
-          end
-        else
-          raise "Received #{key_fields.count} key fields. Only 1/2 supported."
-        end
-      end
-    end
-  end
-end

data/lib/nexpose_servicenow/helpers/connection_helper.rb

@@ -1,84 +0,0 @@
-require_relative '../queries/queries_base'
-
-module NexposeServiceNow
-  class ConnectionHelper
-    NOT_IMPL = 'Error: Method not implemented.'
-
-    def initialize(url, port, username, password, silo='')
-      @log = NexposeServiceNow::NxLogger.instance
-      @url = url
-      @port = port
-      @username = username
-      @password = password
-      @silo = silo
-
-      @timeout = 21600
-    end
-
-    def self.get_report_names(query_name, ids)
-      if QueriesBase.single_report?(query_name)
-        return [ id: -1, report_name: get_report_name(query_name) ]
-      end
-
-      ids.map do |id|
-        { id: id.first, report_name: get_report_name(query_name, id.first) }
-      end
-    end
-
-    def self.get_report_name(query_name, id=nil)
-      name = "Nexpose-ServiceNow-#{query_name}"
-      name += "-#{id}" unless QueriesBase.single_report?(query_name) || id.nil?
-      name
-    end
-
-    def create_query_options(query_options, nexpose_id=nil)
-      options = {}
-      options[:vuln_query_date] = query_options[:vuln_query_date]
-      options[:site_id] = nexpose_id
-      options[:id_type] = 'site'
-      options[:filters] = query_options[:filters] || {}
-      options[:cvss_v] = query_options[:cvss_v]
-
-      # Without a nexpose ID, we don't have a specific delta
-      return options if [nil, -1].include? nexpose_id
-
-      if query_options[:delta_values].empty?
-        error_msg = 'No delta values provided. Exiting...'
-        @log.log_error_message error_msg
-        raise error_msg
-      end
-
-      options[:delta] = "#{query_options[:delta_values][nexpose_id] || 0}"
-
-      @log.log_message("Query options: #{options}")
-
-      options
-    end
-
-    def self.get_filepath(report_name, output_dir)
-      path = File.join output_dir, "#{report_name}.csv"
-      File.expand_path path
-    end
-
-    def connect(username, password)
-      raise NOT_IMPL
-    end
-
-    def generate_report(query_name, ids, id_type, output_dir, query_options={})
-      raise NOT_IMPL
-    end
-
-    # Pulls the collection IDs from Nexpose (e.g. asset groups, sites)
-    def collection_ids(collection_type)
-      raise NOT_IMPL
-    end
-
-    def save_report(report_name, report_id, output_dir)
-      raise NOT_IMPL
-    end
-
-    def get_cvss_version_strings(use_v3)
-      raise NOT_IMPL
-    end
-  end
-end

data/lib/nexpose_servicenow/helpers/data_warehouse_helper.rb

@@ -1,140 +0,0 @@
-require 'pg'
-require_relative './connection_helper'
-require_relative '../queries/warehouse_queries'
-
-module NexposeServiceNow
-  class DataWarehouseHelper < ConnectionHelper
-
-    SSL_MODE = 'prefer'
-    GRANULARITY = 500
-    CHUNK_LIMIT = 4_500_000
-
-    def initialize(url, port, username, password, silo='')
-      super(url, port, username, password, silo)
-    end
-
-    def connect
-      @log.log_message 'Creating DWH connection'
-      PG::Connection.open(:host => @url,
-                          :dbname => @silo,
-                          :port => @port,
-                          :user => @username,
-                          :password => @password,
-                          :sslmode => SSL_MODE)
-    end
-
-    def generate_report(query_name, ids, id_type, output_dir, query_options={})
-      output_dir = File.expand_path(output_dir.to_s)
-
-      #A single report doesn't use site filters
-      ids = [-1] if WarehouseQueries.single_report?(query_name)
-
-      query_options[:cvss_v] = get_cvss_version_strings(query_options[:cvss_v3])
-
-      page_size = query_options[:page_size]
-      row_limit = query_options[:row_limit]
-
-      chunks = []
-
-      base_name = "query_#{query_name}"
-
-      ids.each do |id|
-        delta_options = create_query_options(query_options, id)
-        query = WarehouseQueries.send(query_name, delta_options)
-
-        # Open the CSV file to write as pages are retrieved
-        report_name = self.class.get_report_name(query_name, id)
-        @log.log_message "Running query for #{report_name}"
-
-        local_file_name = self.class.get_filepath(report_name, output_dir)
-        csvFile = File.open(local_file_name, 'wb')
-
-        conn = connect
-        conn.transaction do
-          table_name = "query_#{query_name}"
-          table_name = "#{base_name}_#{id}" if id && id.to_i > 0
-
-          @log.log_message "Creating cursor: #{table_name}"
-          conn.exec("DECLARE #{table_name} CURSOR FOR #{query}")
-          res = conn.exec("FETCH #{page_size} FROM #{table_name}")
-
-          # Headers
-          headers = res.fields.join(',')
-          csvFile.puts(headers)
-
-          # Declare the initial chunk
-          chunk = { start: csvFile.pos, length: 0, row_count: 0 }
-          chunk[:site_id] = id unless id.nil? || id == -1
-
-          # Should we overwrite 'res' to release the memory?
-          all_lines = res.values.map { |r| r.join(',') }
-
-          # Table declared, so keep reading pages until it's consumed
-          data_left = true
-          while(data_left)
-            # Lift out a number of lines in a chunk
-            text = all_lines.slice!(0, GRANULARITY)
-
-            # Try to get the next page
-            if text.nil? || text.count == 0
-              res = conn.exec("FETCH #{page_size} FROM #{table_name}")
-              if res.values.count == 0
-                chunks << chunk
-                break
-              end
-              all_lines = res.values.map { |r| r.join(',') }
-              next
-            end
-
-            # Work out the details for this chunk
-            line_count = text.count
-            text = text.join("\n")
-            text << "\n"
-            byte_size = text.bytesize
-
-            # Test whether limits would be exceeded
-            below_row_limit = chunk[:row_count] + line_count <= row_limit
-            below_size_limit = chunk[:length] + byte_size < CHUNK_LIMIT
-
-            if below_size_limit && below_row_limit
-              chunk[:length] += byte_size
-              chunk[:row_count] += line_count
-            else
-              chunks << chunk.dup
-              # Store the current pos since the next chunk isn't written
-              chunk[:start] = csvFile.pos
-              chunk[:length] = byte_size
-              chunk[:row_count] = line_count
-            end
-
-            csvFile.write(text)
-          end
-        end
-
-        conn.finish
-
-        # Close the file for this specific report
-        csvFile.close
-        @log.log_message "Report generated: #{report_name}"
-      end
-
-      chunks.each { |c| c.delete :row_count }
-
-      @log.log_message "Finished running query: #{query_name}"
-      chunks.to_json
-    end
-
-    def collection_ids
-      @log.log_message 'Retrieving list of site IDs'
-      connection = connect
-      query = 'select site_id from dim_site'
-      result = connection.query(query)
-      result.map { |r| r['site_id'] }
-      connection.finish
-    end
-
-    def get_cvss_version_strings(use_v3)
-      use_v3 ? { choice: '_v3', fallback: '' } : { choice: '', fallback: '' }
-    end
-  end
-end