nexpose_servicenow 0.4.15

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NmE4MDhhMzQ0NDE1ZmZhN2M1NjBjYzI5ODRlYmViMzczOTlkNDRmZg==
+  data.tar.gz: !binary |-
+    NmEyMjJjOGM2YmUzZGRmYThhZGMzNGI5MWFjNGNkMmU3MzYxMjc4Mg==
+SHA512:
+  metadata.gz: !binary |-
+    OTMzZmFmYTM3YjBkYjliZGI4ZWY4Mzk2YTIxYzg2M2QxYjE4Mzg0NDI5NmUx
+    NmJmNGMzYzEzODRkNGI5YTk2ZTJjNTA0NTFjODRlM2E2NjZjZDkyNzZmYjhl
+    MTEwZWRhOWFiZjI0ZTQzZGI5ODZjYzZlZDAzY2U4ZTdlMTI3ZDM=
+  data.tar.gz: !binary |-
+    Y2ZiZTllNWM2ZDgyM2IwOWRkM2RmY2MwNDQ0NjE5NTI3ODM2YzQ2MGZkYzMw
+    M2U0YzdhNGQ4MzI4Mjg2MWM3NjVjOTYxZjI0MjJhOGJhNzkyMmEwOTI3ZjE2
+    NzUxNjg1YjgxMjAwODZiZjVjMDVmODI1Mjc2NjQ0MjIzZDc4MmQ=

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in nexpose_servicenow.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 David Valente
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,16 @@
+# NexposeServicenow
+
+## Installation
+
+gem install nexpose_servicenow
+
+## Usage
+
+## Development
+
+## Contributing
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/nexpose_servicenow

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'nexpose_servicenow'
+
+NexposeServiceNow::Main.start(ARGV)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/nexpose_servicenow.rb

@@ -0,0 +1,173 @@
+require 'csv'
+require 'optparse'
+require 'nexpose'
+require 'uri'
+require 'nexpose_servicenow/queries'
+require 'nexpose_servicenow/nexpose_helper'
+require 'nexpose_servicenow/arg_parser'
+require 'nexpose_servicenow/chunker'
+require 'nexpose_servicenow/nx_logger'
+require 'nexpose_servicenow/historical_data'
+require "nexpose_servicenow/version"
+
+module NexposeServiceNow
+  class Main
+    def self.start(args)
+      options = ArgParser.parse(args)
+
+      log = setup_logging(options)
+      log.log_message("Options: #{options}")
+
+      if options[:nexpose_ids].first.to_s == "0"
+        log.log_message('Retrieving array of all site IDs')
+        options[:nexpose_ids] = get_nexpose_helper(options).all_sites.sort
+      end             
+
+      report_details = NexposeHelper.get_report_names(options[:query], 
+                                                    options[:nexpose_ids])
+      report_details.each do |r|
+        r[:report_name] = NexposeHelper.get_filepath(r[:report_name], 
+                                                     options[:output_dir])
+      end
+
+      update_last_scan_data(options)
+      create_report(report_details, options)  
+
+      log.log_message("Initialising #{options[:mode]} mode")
+      self.send("#{options[:mode]}_mode", report_details, options)
+    end
+
+    def self.get_nexpose_helper(options)
+      NexposeHelper.new(options[:nexpose_url], 
+                        options[:nexpose_port],
+                        options[:nexpose_username],
+                        options[:nexpose_password])
+    end
+
+    def self.setup_logging(options)
+      log = NexposeServiceNow::NxLogger.instance
+      log.setup_statistics_collection(NexposeServiceNow::VENDOR, 
+                                      NexposeServiceNow::PRODUCT, 
+                                      NexposeServiceNow::VERSION)
+      log.setup_logging(true,
+                        options[:log_level] || 'info',
+                        false)
+      log
+    end
+
+    #Merges in the details from the last time the 
+    #integration ran reports.
+    def self.update_last_scan_data(options)
+      return unless options[:mode] == "latest_scans"
+      historical_data = HistoricalData.new(options)
+      historical_data.update_last_scan_data
+    end
+
+    #Create a report if explicitly required or else an existing
+    #report file isn't found
+    def self.create_report(report_details, options)
+      return if options[:mode].start_with? 'update_'
+
+      unless options[:gen_report]
+        #If any file is missing, perform all queries
+        return if report_details.all? { |f| File.exists?(f[:report_name]) } 
+      end
+
+      #Filter it down to sites which actively need queried
+      sites_to_scan = filter_sites(options)
+      nexpose_helper = get_nexpose_helper(options)
+      hist_data = HistoricalData.new(options)
+      vuln_query = options[:query].to_s.start_with? 'vulnerabili'
+      last_run = nil
+      
+      start_time = Time.new
+      query_options = { last_scans: hist_data.last_scan_ids(sites_to_scan) }      
+      query_options[:vuln_query_date] = hist_data.last_vuln_run if vuln_query
+
+      filename = nexpose_helper.create_report(options[:query],
+                                              sites_to_scan,
+                                              options[:id_type],
+                                              options[:output_dir],
+                                              query_options)
+
+      hist_data.create_last_vuln_data(start_time, sites_to_scan) if vuln_query
+
+      #A single String may be returned or an Array of Strings
+      if filename.class.to_s == "Array"
+        filename.map! { |f| File.expand_path(options[:output_dir], f) }
+        return filename.join("\n")
+      end
+
+      File.expand_path(options[:output_dir], filename)
+    end
+
+    def self.filter_sites(options)
+      #These queries always run to make sure certain data is up to date
+      exceptions = ['vulnerabili', 'asset_groups', 'sites', 'tags']
+      if exceptions.any? { |e| options[:query].to_s.start_with? e }
+        return options[:nexpose_ids] 
+      end
+
+      #Always run the query for latest scans or vulnerabilities
+      if options[:mode] == 'latest_scans' || 
+         options[:mode] == 'get_chunk'
+        return options[:nexpose_ids] 
+      end
+
+      historical_data = HistoricalData.new(options)
+      sites_to_scan = historical_data.sites_to_scan
+
+      return sites_to_scan unless (sites_to_scan.nil? || sites_to_scan.empty?)
+
+      log = NexposeServiceNow::NxLogger.instance
+      log.log_message "Sites #{options[:nexpose_ids]} are up to date."
+      log.log_message "Query requested was: #{options[:query]}."
+      exit 0
+    end
+
+    #Print the chunk info
+    def self.chunk_info_mode(report_details, options)
+      chunker = Chunker.new(report_details, options[:row_limit])
+      filtered_sites = filter_sites(options)
+      puts chunker.preprocess(filtered_sites)
+    end
+
+    #Prints a chunk of CSV to the console
+    def self.get_chunk_mode(report_details, options)
+      #Get the byte offset and length
+      chunker = Chunker.new(report_details, options[:row_limit])
+      filtered_sites = filter_sites(options)
+
+      puts chunker.read_chunk(options[:chunk_start],
+                              options[:chunk_length],
+                              filtered_sites.first)
+    end
+
+    def self.latest_scans_mode(report_details, options)
+      historical_data = HistoricalData.new(options)
+      puts historical_data.filter_report
+    end
+
+    def self.remove_last_scan_mode(report_details, options)
+      historical_data = HistoricalData.new(options)
+      historical_data.remove_last_scan_data
+    end
+
+    def self.update_last_scan_mode(report_details, options)
+      historical_data = HistoricalData.new(options)
+      historical_data.set_last_scan(options[:nexpose_ids].first,
+                                    options[:last_scan_data])
+    end
+    
+    def self.remove_last_vuln_mode(report_details, options)
+      historical_data = HistoricalData.new(options)
+      historical_data.remove_last_vuln_data
+    end
+    
+    def self.update_last_vuln_mode(report_details, options)
+      historical_data = HistoricalData.new(options)
+      historical_data.set_last_vuln(options[:last_scan_data],
+                                    options[:nexpose_ids])
+    end
+  end
+end

data/lib/nexpose_servicenow/arg_parser.rb

@@ -0,0 +1,173 @@
+require 'optparse'
+require 'json'
+require_relative './queries'
+require_relative './nx_logger'
+
+module NexposeServiceNow
+  class ArgParser
+    NX_ID_TYPES = %i[site tag]
+    MODES = %i[chunk_info get_chunk latest_scans
+               remove_last_scan update_last_scan
+               remove_last_vuln update_last_vuln]
+
+    QUERY_NAMES = Queries.methods(false)
+    
+    QUERY_ALIASES = { 'devices' => 'cmdb_ci_outofband_device', 
+                      'vuln_items' => 'sn_vul_vulnerable_item', 
+                      'vuln_entries' => 'sn_vul_third_party_entry' }
+     
+    def self.parse(args)
+      options = Hash.new
+
+      opt_parser = OptionParser.new do |opts|
+        opts.banner = "Usage: example.rb [options]"
+
+        opts.on("-o", "--output-dir DIRECTORY",
+                "Directory in which to save reports") do |output_dir|
+          options[:output_dir] = output_dir
+        end
+
+        opts.on("-m", "--mode MODE",
+                "Mode for program output. (#{MODES.join(', ')})") do |mode|
+          options[:mode] = mode
+        end
+
+        opts.on("-g", "--generate-report BOOLEAN",
+                "True to generate and download new report.") do |gen|
+          char = gen.downcase[0]
+          options[:gen_report] = char == 'y' || char == 't'
+        end
+
+        opts.separator ""
+        opts.separator "Query options:"
+
+        opts.on("-q", "--query QUERY", QUERY_NAMES, 
+                "Select query (#{QUERY_NAMES.join(', ')})") do |query| 
+          options[:query] = query
+        end
+
+        opts.on("-t", "--type TYPE", NX_ID_TYPES, 
+                "Select type (#{NX_ID_TYPES.join(', ')})") do |type| 
+          options[:id_type] = type
+        end
+
+        opts.on("-i", "--items x,y,z", Array, 
+                "IDs of the nexpose items to scan") do |items|
+          options[:nexpose_ids] = items
+        end
+
+        opts.separator ""
+        opts.separator "Nexpose options:"
+        
+        opts.on("-n", "--nexpose-address URL", 
+                "URL of the Nexpose server") do |url|
+          port = url.slice!(/:(\d+)$/)
+          port.slice! ':' unless port.nil?
+
+          url.slice! /https:\/\//
+          options[:nexpose_url] = url
+          options[:nexpose_port] = port || '3780'
+          @full_url = options[:nexpose_url] + ':' + options[:nexpose_port]
+        end
+
+        opts.on("-u", "--user USER",
+                "Username for Nexpose console") do |username|
+          options[:nexpose_username] = username
+        end
+
+        opts.on("-p", "--password PASSWORD",
+                "Password for the Nexpose user") do |password|
+          options[:nexpose_password] = password
+        end
+
+        opts.separator ""
+        opts.separator "Chunk info mode options:"
+        
+        opts.on("-r", "--row-limit LIMIT",
+                "Maximum number of rows per chunk (with header).") do |limit|
+          options[:row_limit] = limit.to_i
+          options[:row_limit] = 9_999_999 if options[:row_limit] <= 0
+        end
+
+        opts.separator ""
+        opts.separator "Get chunk mode options:"
+
+        opts.on("-s", "--start START",
+                "The chunk starting offset.") do |start|
+          options[:chunk_start] = start.to_i
+        end
+
+        opts.on("-l", "--length LENGTH",
+                "The chunk length.") do |length|
+          options[:chunk_length] = length.to_i
+        end
+
+        opts.separator ""
+        opts.separator "Last scan file modification options:"
+
+        opts.on("-d", "--data DATA",
+                "Date or scan ID to be inserted in last scan file.") do |data|
+          options[:last_scan_data] = data
+        end
+
+        opts.separator ""
+        opts.separator "Common options:"
+
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+      end 
+
+      opt_parser.parse!(args)
+      options = self.validate_input(options)
+      options = self.get_env_settings(options)
+      options
+    end
+
+    #TODO: Validate input depending on mode AND whether generating a report
+    # is required.
+    def self.validate_input(options)
+      #Insert defaults. Some are mode-specific.
+      options[:output_dir] ||= '.'
+      options[:row_limit] ||= 9_999_999
+      options[:id_type] ||= 'site'
+      options[:nexpose_ids] ||= []
+
+      options[:query] = 'latest_scans' if options[:mode] == 'latest_scans'
+
+      #By default, a report won't be generated if a chunk's being retrieved
+      if options[:gen_report].nil?
+        options[:gen_report] = options[:mode] == "chunk_info" ||
+                               options[:mode] == "latest_scans"
+      end
+
+      options
+    end
+
+    def self.get_env_settings(options)
+      #Only need these if a query is being performed
+      return options if options[:gen_report] == false
+
+      log = NexposeServiceNow::NxLogger.instance
+      log.log_message "Retrieving environment variables."
+
+      # Retrieve environment variable settings
+      %i[url port username password].each do |setting|
+        option = "nexpose_#{setting}"
+        setting = ENV[option.upcase]
+        sym = option.intern
+        options[sym] ||= setting
+
+        if options[sym].nil?
+          error = "Option #{sym} wasn't supplied."
+          log.log_error_message error
+          $stderr.puts "ERROR: #{error}"
+          exit -1
+        end
+      end
+
+      options
+    end
+  end
+end		

data/lib/nexpose_servicenow/chunker.rb

@@ -0,0 +1,106 @@
+
+module NexposeServiceNow
+  class Chunker
+    def initialize(report_details, row_limit)
+      @row_limit = row_limit
+      @size_limit = 4_500_000
+      @report_details = report_details
+      @header = get_header
+
+      setup_logging
+    end
+
+    def setup_logging
+      @log = NexposeServiceNow::NxLogger.instance
+      @log.log_message("Chunker File Limit: #{@size_limit}MB");
+      @log.log_message("Chunk Row Limit: #{@row_limit}")
+    end
+
+    #Grab the header from the first file
+    def get_header
+      file = File.open(@report_details.first[:report_name], "r")
+      header = file.readline
+      file.close
+
+      header
+    end
+
+    def preprocess(nexpose_ids=nil)
+      @log.log_message("Breaking file #{@file_path} down into chunks.")
+
+      all_chunks = []
+      @report_details.each do |report|
+        chunks = process_file(report[:report_name], report[:id])
+        all_chunks.concat chunks
+      end 
+
+      @log.log_message("Files broken down into #{all_chunks.count} chunks")
+
+      puts all_chunks.to_json
+    end
+
+    def process_file(file_path, site_id=nil)
+      relative_size_limit = @size_limit - @header.bytesize
+      chunk = { site_id: site_id,
+                start: @header.bytesize, 
+                length: 0, 
+                row_count: 0 }
+      
+      chunks = []
+      csv_file = CSV.open(file_path, "r", headers: true)
+      while(true)
+        position = csv_file.pos
+        line = csv_file.shift
+        row_length = line.to_s.bytesize
+        
+        if line.nil?
+          chunks << chunk
+          break
+        elsif chunk[:length]+row_length < relative_size_limit &&
+              chunk[:row_count] + 1 < @row_limit
+          chunk[:length] += row_length
+          chunk[:row_count] += 1
+        else
+          chunks << chunk
+
+          #Initialise chunk with this row information
+          chunk = { site_id: site_id,
+                    start: position, 
+                    length: row_length, 
+                    row_count: 1 }
+        end  
+      end
+      csv_file.close
+
+      #Should we include the row count?
+      chunks.each do |c| 
+        c.delete :row_count
+
+        #Should we do this...?
+        c.delete :site_id if c[:site_id].nil? || c[:site_id] == -1
+      end
+
+      chunks
+    end
+
+    def get_file(site_id=nil)
+      #-1 indicates a single query report
+      return @report_details.first[:report_name] if site_id.to_i <= 0
+
+      report = @report_details.find { |r| r[:id].to_s == site_id.to_s }
+      report[:report_name]
+    end
+
+    def read_chunk(start, length, site_id=nil)
+      @log.log_message("Returning chunk. Start: #{start}, Length: #{length}, File: #{@file_path}")
+
+      #If the header isn't in the chunk, prepend it
+      header = start == 0 ? "" : @header
+
+      file = File.open(get_file(site_id), "rb")
+      file.seek(start)
+      puts header + file.read(length)
+      file.close
+    end
+  end
+end