nexpose_servicenow 0.7.3 → 0.8.0

data/lib/nexpose_servicenow/helpers/nexpose_console_helper.rb

@@ -1,164 +0,0 @@
-require 'nexpose'
-require 'fileutils'
-require_relative './connection_helper'
-require_relative '../queries/nexpose_queries'
-require_relative '../nx_logger'
-require_relative '../csv_compare'
-
-module NexposeServiceNow
-  class NexposeConsoleHelper < ConnectionHelper
-    def initialize(url, port, username, password, silo='')
-      super(url, port, username, password, silo)
-
-      @nsc = connect
-    end
-
-    def generate_report(query_name, ids, id_type, output_dir, query_options={})
-      output_dir = File.expand_path(output_dir.to_s)
-
-      #A single report doesn't use site filters        
-      ids = [-1] if NexposeQueries.single_report?(query_name)
-
-      #If running latest_scans, send up information
-      if query_name == 'latest_scans'
-        @log.on_connect(@url, @port, @nsc.session_id, '{}')
-      end
-
-      query_options[:cvss_v] = get_cvss_version_strings(query_options[:cvss_v3])
-
-      ids.each do |id|
-        report_name = self.class.get_report_name(query_name, id)
-        clean_up_reports(report_name)
-
-        delta_options = create_query_options(query_options, id)
-
-        query = NexposeQueries.send(query_name, delta_options)
-
-        report_id = generate_config(query, report_name, [id], id_type)
-
-        run_report(report_id, report_name)
-        local_report_name = save_report(report_name, report_id, output_dir)
-
-        if NexposeQueries.csv_diff_required?(query_name)
-          @log.log_message "Calculating diff for #{local_report_name}..."
-          CsvCompare.update_report_with_diff(local_report_name,
-                                             NexposeQueries.query_keys(query_name))
-        end
-      end
-
-      nil
-    end
-
-    def clean_up_reports(report_name)
-      reports = @nsc.list_reports
-      reports.select! { |r| r.name.start_with? report_name }
-      reports.each { |r| @nsc.delete_report_config(r.config_id) }
-    end
-
-    def generate_config(query, report_name, ids, id_type)
-      @nsc = connect
-      @log.log_message "Generating report config with name #{report_name}..."
-      report_config = Nexpose::ReportConfig.new(report_name, nil, 'sql')
-      report_config.add_filter('version', '2.0.1')
-      report_config.add_filter('query', query)
-
-      id_type = id_type.to_s.split('_').last
-      ids.each { |id| report_config.add_filter(id_type, id) unless id == -1 }
-
-
-      @log.log_message "Saving report config #{report_name}..."
-      report_id = report_config.save(@nsc, false)
-      @log.log_message "Report #{report_name} saved"
-
-      report_id
-    end
-
-    def run_report(report_id, report_name)
-      @log.log_message "Running Report #{report_name}, ID: #{report_id}..."
-      @nsc.generate_report(report_id, false)
-      wait_for_report(report_id)
-    end
-
-    def wait_for_report(id)
-      wait_until(:fail_on_exceptions => true, :on_timeout => "Report generation timed out. Status: #{r = @nsc.last_report(id); r ? r.status : 'unknown'}") {
-        if %w(Failed Aborted Unknown).include?(@nsc.last_report(id).status)
-          raise "Report failed to generate! Status <#{@nsc.last_report(id).status}>"
-        end
-        @nsc.last_report(id).status == 'Generated'
-      }
-    end
-
-    def wait_until(options = {})
-      polling_interval = 15
-      time_limit = Time.now + @timeout
-      loop do
-        begin
-          val = yield
-          return val if val
-        rescue Exception => error
-          @log.log_error_message("Error during wait_until: #{error}")
-          raise error if options[:fail_on_exceptions]
-        end
-        if Time.now >= time_limit
-          if options[:on_timeout]
-            @log.log_error_message("#{options[:on_timeout]}. Exiting...")
-            raise options[:on_timeout]
-          end
-          error ||= 'Timed out waiting for condition.'
-          @log.log_error_message("#{error}. Exiting...")
-          raise error
-        end
-        sleep polling_interval
-      end
-    end
-
-    def save_report(report_name, report_id, output_dir)
-      @log.log_message "Saving report - Name: #{report_name}, ID: #{report_id}..."
-      local_file_name = self.class.get_filepath(report_name, output_dir)
-      File.delete(local_file_name) if File.exists? local_file_name
-
-      #log 'Downloading report...'
-      report_details = @nsc.last_report(report_id)
-      File.open(local_file_name, 'wb') do |f| 
-        f.write(@nsc.download(report_details.uri))
-      end
-      
-      begin
-        # Refresh the connection
-        @nsc = connect
-
-        # Got the report, cleanup server-side
-        @nsc.delete_report_config(report_id)
-      rescue
-        @log.log_error_message 'Error deleting report'
-      end
-
-      local_file_name
-    end
-
-    def connect
-      begin
-        connection = Nexpose::Connection.new(@url, @username, @password, @port)
-        connection.login
-        @log.log_message 'Logged in.'
-      rescue Exception => e
-        msg = "ERROR: Could not log in. Check log and settings.\n#{e}"
-        @log.log_error_message msg
-        $stderr.puts msg
-        exit -1
-      end
-
-      connection
-    end
-
-    # Pulls the collection IDs from Nexpose (e.g. asset groups, sites)
-    def collection_ids(collection_type)
-      @nsc.send("#{collection_type}s").map { |s| s.id }.sort
-    end
-
-    def get_cvss_version_strings(use_v3)
-      return { choice: '_v3', vector: '_v3', fallback: '' } if use_v3
-      { choice: '_v2', vector: '', fallback: '' }
-    end
-  end
-end

data/lib/nexpose_servicenow/historical_data.rb

@@ -1,102 +0,0 @@
-require_relative './nx_logger'
-
-module NexposeServiceNow
-  class HistoricalData
-
-    SITE_IDENTIFIER = 'site_id'
-    SITE_DELTA_VALUE = 'last_scan_id'
-    SITE_TIMESTAMP_VALUE = 'finished'
-
-    DEFAULT_TIMESTAMP_VALUE = '1985-01-01 12:00:00'
-
-    GROUP_FILE_PATTERN = 'Nexpose-ServiceNow-asset_group*.csv'
-
-    def initialize(output_dir)
-      local_dir = File.expand_path(output_dir)
-      @remote_file = File.join(local_dir, 'Nexpose-ServiceNow-latest_scans.csv')
-      @log = NexposeServiceNow::NxLogger.instance
-    end
-
-    # Filters out irrelevant site IDs
-    def filter_ids(site_ids)
-      original_site_ids = site_ids.keys
-      default_values = ['0', "\"#{DEFAULT_TIMESTAMP_VALUE}\""]
-
-      # Reject if the delta value matches a default value
-      site_ids.reject! do |site, delta|
-        default_values.any? { |v| delta == v }
-      end
-
-      # Get a list of excluded sites
-      excluded = original_site_ids - site_ids.keys
-
-      if excluded.count > 0
-        @log.log_message("Excluding #{excluded} from old vulns query.")
-      end
-
-      site_ids
-    end
-
-    # Replaces scan IDs of sites that aren't to be imported with 0
-    def filter_report(site_ids=[])
-      @log.log_message 'Filtering report down to sites which will be queried'
-
-      remote_csv = load_scan_id_report
-      nexpose_ids = site_ids.map(&:to_s)
-
-      return remote_csv if nexpose_ids.first == '0'
-
-      remote_csv.each do |row|
-        unless nexpose_ids.include?(row[SITE_IDENTIFIER])
-          row[SITE_DELTA_VALUE] = '0'
-          row[SITE_TIMESTAMP_VALUE] = DEFAULT_TIMESTAMP_VALUE
-        end
-      end
-
-      remote_csv
-    end
-
-    # Reads the downloaded report containing LATEST scan IDs
-    def load_scan_id_report
-      @log.log_message 'Loading scan data report'
-      unless File.exists? @remote_file
-        error = 'Latest scan report file could not be found.'
-        @log.log_message error
-        raise error
-      end
-      CSV.read(@remote_file, headers: true)
-    end
-
-    #########################################################
-    #                    Experimental                       #
-    #########################################################
-
-    # These should probably return strings that can be mlog'd
-    def log_and_print(message)
-      puts message
-      @log.log_message message unless @log.nil?
-    end
-
-    def log_and_error(message)
-      $stderr.puts "ERROR: #{message}"
-      @log.log_error_message message unless @log.nil?
-    end
-
-    # Deletes all of the CSV files matching the pattern
-    def remove_diff_files(output_dir)
-      local_path = File.expand_path(output_dir)
-      group_csv_files = Dir.glob(File.join(local_path, GROUP_FILE_PATTERN))
-
-      group_csv_files.each do |file|
-        begin
-          File.delete file
-          log_and_print "File #{file} deleted."
-        rescue Exception => e
-          log_and_error "Error removing file:\n#{e}"
-        end
-      end
-    end
-  end
-end
-
-

data/lib/nexpose_servicenow/nx_logger.rb

@@ -1,166 +0,0 @@
-require 'fileutils'
-require 'json'
-require 'net/http'
-require 'singleton'
-
-module NexposeServiceNow
-  class NxLogger
-    include Singleton
-    LOG_PATH = "./logs/rapid7_%s.log"
-    KEY_FORMAT = "external.integration.%s"
-    PRODUCT_FORMAT = "%s_%s"
-
-    DEFAULT_LOG = 'integration'
-    PRODUCT_RANGE = 4..30
-    KEY_RANGE = 3..15
-
-    ENDPOINT = '/data/external/statistic/'
-
-    def initialize()
-      create_calls
-      @logger_file = get_log_path @product
-      setup_logging(true, 'info')
-    end
-
-    def setup_statistics_collection(vendor, product_name, gem_version)
-      begin
-        @statistic_key = get_statistic_key vendor
-        @product = get_product product_name, gem_version
-      rescue => e
-        #Continue
-      end
-    end
-
-    def setup_logging(enabled, log_level = 'info', stdout=false)
-      @stdout = stdout
-
-      log_message('Logging disabled.') unless enabled || @log.nil?
-      @enabled = enabled
-      return unless @enabled
-
-      @logger_file = get_log_path @product
-
-      require 'logger'
-      directory = File.dirname(@logger_file)
-      FileUtils.mkdir_p(directory) unless File.directory?(directory)
-      io = IO.for_fd(IO.sysopen(@logger_file, 'a'), 'a')
-      io.autoclose = false
-      io.sync = true
-      @log = Logger.new(io, 'weekly')
-      @log.level = if log_level.to_s.casecmp('info') == 0 
-                     Logger::INFO 
-                   else
-                     Logger::DEBUG
-                   end
-      log_message("Logging enabled at level <#{log_level}>")
-    end
-
-    def create_calls
-      levels = [:info, :debug, :error, :warn]
-      levels.each do |level|
-        method_name = 
-        define_singleton_method("log_#{level.to_s}_message") do |message|
-          puts message if @stdout
-          @log.send(level, message) unless !@enabled || @log.nil?
-        end
-      end
-    end
-
-    def log_message(message)
-      log_info_message message
-    end
-
-    def log_stat_message(message)
-    end
-
-    def get_log_path(product)
-      product.downcase! unless product.nil?
-      File.join(File.dirname(__FILE__), LOG_PATH % (product || DEFAULT_LOG))
-    end
-
-    def get_statistic_key(vendor)
-      if vendor.nil? || vendor.length < KEY_RANGE.min
-        log_stat_message("Vendor length is below minimum of <#{KEY_RANGE}>")
-        return nil
-      end
-
-      vendor.gsub!('-', '_')
-      vendor.slice! vendor.rindex('_') until vendor.count('_') <= 1
-
-      vendor.delete! "^A-Za-z0-9\_"
-
-      KEY_FORMAT % vendor[0...KEY_RANGE.max].downcase
-    end
-
-    def get_product(product, version)
-      return nil if ((product.nil? || product.empty?) || 
-                     (version.nil? || version.empty?))
-
-      product.gsub!('-', '_')
-      product.slice! product.rindex('_') until product.count('_') <= 1
-
-      product.delete! "^A-Za-z0-9\_"
-      version.delete! "^A-Za-z0-9\.\-"
-
-      product = (PRODUCT_FORMAT % [product, version])[0...PRODUCT_RANGE.max]
-
-      product.slice! product.rindex(/[A-Z0-9]/i)+1..-1
-
-      if product.length < PRODUCT_RANGE.min
-        log_stat_message("Product length below minimum <#{PRODUCT_RANGE.min}>.")
-        return nil
-      end
-      product.downcase
-    end
-
-    def generate_payload(statistic_value='')
-      product_name, separator, version = @product.to_s.rpartition('_')
-      payload_value = {'version' => version}.to_json
-
-      payload = {'statistic-key' => @statistic_key.to_s,
-                 'statistic-value' => payload_value,
-                 'product' => product_name}
-      JSON.generate(payload)
-    end
-
-    def send(nexpose_address, nexpose_port, session_id, payload)
-      header = {'Content-Type' => 'application/json',
-                'nexposeCCSessionID' => session_id,
-                'Cookie' => "nexposeCCSessionID=#{session_id}"}
-      req = Net::HTTP::Put.new(ENDPOINT, header)
-      req.body = payload
-      http_instance = Net::HTTP.new(nexpose_address, nexpose_port)
-      http_instance.use_ssl = true
-      http_instance.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      response = http_instance.start { |http| http.request(req) }
-      log_stat_message "Received code #{response.code} from Nexpose console."
-      log_stat_message "Received message #{response.msg} from Nexpose console."
-      log_stat_message 'Finished sending statistics data to Nexpose.'
-
-      response.code
-    end
-
-    def on_connect(nexpose_address, nexpose_port, session_id, value)
-      log_stat_message 'Sending statistics data to Nexpose'
-
-      if @product.nil? || @statistic_key.nil?
-        log_stat_message('Invalid product name and/or statistics key.')
-        log_stat_message('Statistics collection not enabled.')
-        return
-      end
-      
-      begin
-        payload = generate_payload value
-        send(nexpose_address, nexpose_port, session_id, payload)
-      rescue => e
-        #Let the program continue
-      end
-    end
-
-    #Used by net library for debugging 
-    def <<(value)
-      log_debug_message(value)
-    end
-
-  end
-end

data/lib/nexpose_servicenow/queries/nexpose_queries.rb

@@ -1,459 +0,0 @@
-require_relative './queries_base'
-
-module NexposeServiceNow
-  class NexposeQueries < QueriesBase
-    def self.vulnerabilities(options={})
-      "SELECT
-        concat('R7_', vulnerability_id) as ID,
-        cve.ref as CVE,
-        cwe.ref as CWE,
-        concat('Rapid7 Nexpose') as Source,
-        to_char(date_published, 'yyyy-MM-dd hh:mm:ss') as date_published,
-        to_char(date_modified, 'yyyy-MM-dd hh:mm:ss') as Last_Modified,
-        dvc.category,
-        severity as Severity_Rating,
-        severity_score as Severity,
-        pci_status,
-        pci_adjusted_cvss_score as PCI_Severity,
-        title as Summary,
-        proofAsText(description) as Threat,
-        ROUND(riskscore::numeric, 2) as Riskscore,
-        coalesce(cvss#{options[:cvss_v][:vector]}_vector,
-                 cvss#{options[:cvss_v][:fallback]}_vector) as cvss_vector,
-        ROUND(coalesce(cvss#{options[:cvss_v][:choice]}_impact_score::numeric,
-                       cvss#{options[:cvss_v][:fallback]}_impact_score::numeric),
-              2) as Impact_Score,
-        ROUND(coalesce(cvss#{options[:cvss_v][:choice]}_exploit_score::numeric,
-                       cvss#{options[:cvss_v][:fallback]}_exploit_score::numeric),
-              2) as Exploit_Score,
-        cvss_access_complexity_id as Access_Complexity,
-        cvss_access_vector_id as Access_Vector,
-        cvss_authentication_id as Authentication,
-        ROUND(coalesce(cvss#{options[:cvss_v][:choice]}_score::numeric,
-                       cvss#{options[:cvss_v][:fallback]}_score::numeric),
-              2) as Vulnerability_Score,
-        cvss_integrity_impact_id as Integrity_Impact,
-        cvss_confidentiality_impact_id as Confidentiality_Impact,
-        cvss_availability_impact_id as Availability_Impact,
-        CAST(CASE
-               WHEN exploits > 0
-                 THEN 1
-               ELSE 0
-             END AS bit) as Exploitability,
-        CAST(CASE
-               WHEN malware_kits > 0
-                 THEN 1
-               ELSE 0
-             END AS bit) as Malware_Kits,
-        array_to_string(sol.solutions, ',', '') as Solution
-
-      FROM
-        dim_vulnerability
-
-      LEFT OUTER JOIN
-      (SELECT DISTINCT on(vulnerability_id)
-         vulnerability_id,
-         dvr.reference as ref
-       FROM dim_vulnerability_reference dvr
-       WHERE source='CWE'
-       GROUP BY dvr.vulnerability_id, dvr.reference
-       ) cwe USING (vulnerability_id)
-
-      LEFT OUTER JOIN
-      (SELECT DISTINCT on (vulnerability_id)
-       vulnerability_id,
-       dvr.reference as ref
-       FROM dim_vulnerability_reference dvr
-       WHERE source='CVE'
-       GROUP BY dvr.vulnerability_id, dvr.reference
-      ) cve USING (vulnerability_id)
-
-      LEFT OUTER JOIN(SELECT DISTINCT on (dvc.vulnerability_id) dvc.vulnerability_id, dvc.category_name as category
-        FROM dim_vulnerability_category dvc
-        GROUP BY dvc.vulnerability_id, dvc.category_name) dvc USING (vulnerability_id)
-
-      LEFT OUTER JOIN(SELECT dvr.vulnerability_id, string_agg(dvr.source || ': ' || dvr.reference, '|') as references
-        FROM dim_vulnerability_reference dvr
-        GROUP BY dvr.vulnerability_id) ref USING (vulnerability_id)
-
-      LEFT OUTER JOIN(SELECT vulnerability_id,
-                        array_agg(solution_id) as solutions
-        FROM dim_vulnerability_solution
-        GROUP BY vulnerability_id) sol USING (vulnerability_id)
-      WHERE date_modified >= '#{options[:vuln_query_date]}'"
-    end
-
-
-    def self.vulnerability_references(options={})
-      "SELECT concat('R7_', vulnerability_id) as ID, dvr.Source, dvr.Reference
-        FROM dim_vulnerability
-        JOIN
-            (SELECT vulnerability_id, dvr.Source, dvr.Reference
-            FROM dim_vulnerability_reference dvr) dvr USING (vulnerability_id)
-        WHERE date_modified >= '#{options[:vuln_query_date]}'"
-    end
-
-    def self.vulnerability_category(options={})
-      "SELECT concat('R7_', vulnerability_id) as ID, dvc.Category
-        FROM dim_vulnerability
-        LEFT OUTER JOIN
-                (SELECT vulnerability_id, category_name as Category
-                FROM dim_vulnerability_category dvc) dvc USING (vulnerability_id)
-        WHERE date_modified >= '#{options[:vuln_query_date]}'"
-    end
-    
-    # Filter by site.
-    def self.assets(options={})
-      "SELECT coalesce(host_name, CAST(dim_asset.asset_id as text)) as Name,
-        dim_asset.ip_address,
-        dim_asset.mac_address,
-        concat('Rapid7 Nexpose') as Discovery_Source,
-                CAST(CASE 
-                     WHEN dim_host_type.description = 'Virtual Machine' or dim_host_type.description = 'Hypervisor'
-                       THEN 1 
-                     ELSE 0 
-                   END AS bit) as Is_Virtual,
-        dim_operating_system.description as Operating_System,
-        fact_asset.scan_finished as Most_Recent_Discovery,
-        dim_asset.asset_id as Nexpose_ID,
-        fact_asset.pci_status,
-        concat(fact_asset.aggregated_credential_status_id, ' - ', aggregated_credential_status_description)  as Credential_Status
-
-      FROM dim_asset
-      JOIN fact_asset USING (asset_id)
-      JOIN dim_aggregated_credential_status USING (aggregated_credential_status_id)
-      LEFT OUTER JOIN dim_operating_system on dim_asset.operating_system_id = dim_operating_system.operating_system_id
-      LEFT OUTER JOIN dim_host_type USING (host_type_id)"
-    end
-
-    def self.software_instance(options={})
-      "SELECT asset_id as Nexpose_ID, CAST(da.asset_id as text) as Installed_On, ds.name, ds.vendor, ds.family, ds.version, ds.cpe
-      FROM fact_asset_scan_software
-      LEFT OUTER JOIN (SELECT software_id, name, vendor, family, version, cpe FROM dim_software) ds USING (software_id)
-      LEFT OUTER JOIN (SELECT asset_id, host_name FROM dim_asset) da USING (asset_id)
-      WHERE scan_id = lastScan(asset_id)"
-    end
-
-    def self.service_definition(options={})
-      "SELECT DISTINCT on(dsf.name, ds.name, dp.name, port)
-        dsf.name, ds.name as service_name, dp.name as protocol, port
-
-        FROM (SELECT service_id, protocol_id, port, service_fingerprint_id
-            FROM fact_asset_scan_service
-            GROUP BY asset_id, service_id, protocol_id, port, service_fingerprint_id
-            HAVING min(scan_id) > #{options[:delta]} and max(scan_id) = lastScan(asset_id)) fass
-        JOIN dim_service ds USING (service_id)
-        JOIN dim_protocol dp USING (protocol_id)
-        JOIN dim_service_fingerprint dsf USING (service_fingerprint_id)"
-    end
-
-    # When a service only appears in either the old or latest import,
-    # then it needs deleted or inserted, respectively.
-    def self.service_instance(options={})
-      "SELECT asset_id, dsf.name as name,
-              ds.name as service_name,
-              dp.name as protocol, port,
-            CASE
-              WHEN scan_id = #{options[:delta]}
-                THEN 'old'
-              WHEN scan_id = lastScan(asset_id)
-                THEN 'new'
-              ELSE 'current'
-            END as status
-
-        FROM (SELECT asset_id, service_id, protocol_id, port, min(scan_id) as scan_id, service_fingerprint_id
-              FROM fact_asset_scan_service
-              WHERE scan_id = lastScan(asset_id) OR scan_id = #{options[:delta]}
-              GROUP BY asset_id, service_id, protocol_id, port, service_fingerprint_id
-              HAVING min(scan_id) = max(scan_id)) fass
-        JOIN dim_service ds USING (service_id)
-        JOIN dim_protocol dp USING (protocol_id)
-        JOIN dim_service_fingerprint dsf USING (service_fingerprint_id)
-        GROUP BY asset_id, dsf.name, ds.name, dp.name, port, scan_id"
-    end
-
-    # Need to wipe table each time
-    def self.group_accounts(options={})
-      'SELECT asset_id as Nexpose_ID, daga.name as Group_Account_Name
-        FROM dim_asset
-        JOIN dim_asset_group_account daga USING (asset_id)'
-    end
-
-    # Need to wipe table each time
-    def self.user_accounts(options={})
-      'SELECT da.asset_id as Nexpose_ID, daua.name as User_Account_Name,
-        daua.full_name as User_Account_Full_Name
-
-        FROM dim_asset da
-        JOIN dim_asset_user_account daua USING (asset_id)'
-    end
-
-    def self.asset_groups(options={})
-      'SELECT asset_group_id, name as asset_group_name, description, dynamic_membership
-        FROM dim_asset_group'
-    end
-
-    def self.asset_group_memberships(options={})
-      'select * from dim_asset_group_asset'
-    end
-
-    # Need to wipe table each time
-    def self.sites(options={})
-      'SELECT asset_id as Nexpose_ID, ds.name as site_name
-        FROM dim_asset
-        JOIN dim_site_asset dsa USING (asset_id)
-        JOIN dim_site ds on dsa.site_id = ds.site_id
-        ORDER BY ip_address'
-    end
-
-    # Need to wipe table each time
-    def self.tags(options={})
-      'SELECT asset_id as Nexpose_ID, dt.tag_name
-        FROM dim_tag_asset dta
-        JOIN dim_tag dt on dta.tag_id = dt.tag_id'
-    end
-
-    def self.generate_cve_filter(cves)
-      return '' if cves == nil || cves.empty?
-
-      cves = cves.map { |c| "reference='#{c}'" }.join(' OR ')
-
-      "JOIN (SELECT vulnerability_id, reference
-             FROM dim_vulnerability_reference
-             WHERE #{cves}) dvr USING (vulnerability_id)"
-    end
-
-    #TODO make sure that for max date first_discovered < date2
-    def self.generate_date_filter(dates, table_join=true)
-      return '' if dates == nil
-
-      # No filters applied, so no need to filter
-      return '' if dates.all? { |d| d == nil || d == '' }
-
-      min_date = dates.first
-      max_date = dates.last
-
-      # Version of vulnerable new items
-      unless table_join
-        filters = []
-        filters << "first_discovered >= '#{min_date}'" unless min_date.nil?
-        filters << "first_discovered <= '#{max_date}'" unless max_date.nil?
-        filters = filters.join(' AND ')
-        filters = "WHERE #{filters}"
-
-        return filters
-      end
-
-      date_filters = []
-
-      unless min_date.nil?
-        date_filters << "scan_finished > '#{min_date}'"
-      end
-      unless max_date.nil?
-        date_filters << "scan_started < '#{max_date}'"
-      end
-
-      condition = date_filters.join(' AND ')
-      "JOIN (SELECT scan_id, asset_id
-                           FROM fact_asset_scan
-                           WHERE #{condition}) fas
-                           ON fas.asset_id = da.asset_ID AND
-                           fas.scan_id = first_found"
-    end
-
-    def self.generate_cvss_filter(cvss_range, cvss_strings)
-      return '' if cvss_range.nil? || cvss_range.last.nil?
-
-      cvss_min = cvss_range.first
-      cvss_max = cvss_range.last
-
-      # No need to join if not applying a filter
-      return '' if cvss_min.to_s == '0' && cvss_max.to_s == '10'
-
-      cvss_score = "(coalesce(cvss#{cvss_strings[:choice]}_score,
-                      cvss#{cvss_strings[:fallback]}_score))"
-
-      "JOIN (SELECT vulnerability_id
-             FROM dim_vulnerability
-             WHERE #{cvss_score} >= #{cvss_min} AND #{cvss_score} <= #{cvss_max}) dv
-       USING (vulnerability_id)"
-    end
-
-    def self.generate_cvss_table(cvss_range, cvss_strings)
-      return '' if cvss_range.nil? || cvss_range.last.nil?
-
-      cvss_min = cvss_range.first
-      cvss_max = cvss_range.last
-
-      return '' if cvss_min.to_s == '0' && cvss_max.to_s == '10'
-
-      cvss_score = "(coalesce(cvss#{cvss_strings[:choice]}_score,
-                      cvss#{cvss_strings[:fallback]}_score))"
-
-      "vulns_cvss AS (
-          SELECT vulnerability_id FROM dim_vulnerability
-          WHERE #{cvss_score} >= #{cvss_min} AND #{cvss_score} <= #{cvss_max})"
-    end
-
-    def self.vulnerable_new_items(options={})
-      date_filter = self.generate_date_filter(options[:filters][:date], false)
-
-      cvss_table = self.generate_cvss_table(options[:filters][:cvss],
-                                            options[:cvss_v])
-      cvss_filter = ''
-      if cvss_table != ''
-        cvss_table = ",#{cvss_table}"
-        cvss_filter = 'WHERE EXISTS (
-                       SELECT 1 FROM vulns_cvss vc
-                       WHERE nv.vulnerability_id = vc.vulnerability_id)'
-      end
-
-      "WITH assets AS (
-          SELECT * FROM dim_site_asset
-          WHERE site_id=#{options[:site_id]}
-        ), previous_scan AS (
-          SELECT asset_id, vulnerability_id
-          FROM fact_asset_scan_vulnerability_finding
-          WHERE scan_id=#{options[:delta]}
-        ), new_vulns AS (
-          SELECT asset_id, vulnerability_id, vulnerability_instances
-          FROM assets a
-          JOIN fact_asset_vulnerability_finding favf USING (asset_id)
-          WHERE NOT EXISTS (
-            SELECT 1
-            FROM previous_scan p
-            WHERE favf.asset_id = p.asset_id and favf.vulnerability_id = p.vulnerability_id)
-        )#{cvss_table}
-        ,vuln_instances AS (
-          SELECT asset_id, vulnerability_id, vulnerability_instances,
-                 status_id, proofAsText(proof) as proof, port, protocol_id
-          FROM new_vulns nv
-          JOIN fact_asset_vulnerability_instance USING (asset_id, vulnerability_id)
-          #{cvss_filter}
-        )
-
-        SELECT asset_id as Configuration_Item,
-               TRUE as Active,
-               concat('R7_', vulnerability_id) as Vulnerability,
-               ip_address as IP_Address,
-               first_discovered as First_Found,
-               most_recently_discovered as Last_Found,
-               vulnerability_instances as Times_Found,
-               string_agg(CONCAT('\"', proof ,'\"'), ',') as proof,
-               string_agg(DISTINCT port::character, ',') as ports,
-               string_agg(DISTINCT dp.description, ',') as protocol,
-               array_to_string(dvsol.solution_ids, ',', '') as Solutions,
-               string_agg(DISTINCT dvs.description, ',') as Status
-        FROM vuln_instances
-        JOIN dim_protocol dp USING (protocol_id)
-        JOIN dim_asset USING (asset_id)
-        JOIN dim_vulnerability_status dvs USING (status_id)
-        JOIN (SELECT asset_id, vulnerability_id,
-                     first_discovered, most_recently_discovered
-              FROM fact_asset_vulnerability_age #{date_filter}) fasva USING (asset_id, vulnerability_id)
-        LEFT JOIN (SELECT asset_id, vulnerability_id,
-                          array_agg(DISTINCT solution_id) as solution_ids
-                   FROM dim_asset_vulnerability_solution
-                   GROUP BY asset_id, vulnerability_id) dvsol USING (asset_id, vulnerability_id)
-        GROUP by asset_id, vulnerability_id, first_discovered, ip_address, most_recently_discovered, vulnerability_instances, dvsol.solution_ids
-      "
-    end
-
-    def self.vulnerable_old_items(options={})
-      standard_filter = if options[:id_type] == 'site'
-                          "MAX(fasv.scan_id) >= #{options[:delta]}"
-                        else
-                          "MAX(fasv.scan_id) >= scanAsOf(fasv.asset_id, '#{options[:delta]}') AND
-                           lastScan(fasv.asset_id) > scanAsOf(fasv.asset_id, '#{options[:delta]}')"
-                        end
-
-      cve_filter = self.generate_cve_filter(options[:filters][:cve])
-      date_filter = self.generate_date_filter(options[:filters][:date])
-      cvss_filter = self.generate_cvss_filter(options[:filters][:cvss],
-                                              options[:cvss_v])
-
-      # Only perform this operation is necessary
-      date_field = if date_filter.nil? || date_filter == ''
-                     ''
-                   else
-                     'MIN(fasv.scan_id) as first_found,'
-                   end
-
-      "SELECT
-      CAST(da.asset_id as text) Configuration_Item,
-      FALSE as Active,
-      concat('R7_', subq.vulnerability_id) as Vulnerability
-      FROM (
-             SELECT fasv.asset_id, fasv.vulnerability_id,
-                     #{date_field}
-                     MAX(fasv.scan_id) as latest_found,
-                     s.current_scan
-             FROM fact_asset_scan_vulnerability_finding fasv
-              #{cve_filter}
-              #{cvss_filter}
-              JOIN (
-                 SELECT asset_id, lastScan(asset_id) AS current_scan FROM dim_asset
-              ) s ON s.asset_id = fasv.asset_id
-              GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan
-              HAVING MAX(fasv.scan_id) < current_scan
-                     AND #{standard_filter}
-             ) subq
-      JOIN dim_asset da ON subq.asset_id = da.asset_id
-      #{date_filter}"
-
-    end
-
-    def self.vulnerability_solutions(options={})
-      "SELECT DISTINCT (solution_id)
-         solution_id,
-         nexpose_id,
-         coalesce(NULLIF(CONCAT('\"', proofAsText(fix),'\"'), '\"\"'), 'None') as fix,
-         estimate,
-         summary,
-         solution_type,
-         applies_to,
-         url,
-         coalesce(NULLIF(CONCAT('\"',proofAsText(additional_data),'\"'), '\"\"'), 'None') as additional_data,
-         array_to_string(req_solutions, ',', '') as required_solutions,
-         array_to_string(super_solutions, ',', '') as superceding_solutions
-       FROM dim_solution
-       RIGHT OUTER JOIN (
-           SELECT DISTINCT (solution_id) solution_id
-            FROM (
-              SELECT solution_id, vulnerability_id, date_modified
-              FROM dim_vulnerability
-              LEFT JOIN dim_vulnerability_solution idvs USING (vulnerability_id)
-            ) dvs
-            WHERE date_modified >= '#{options[:vuln_query_date]}'
-            UNION
-            SELECT DISTINCT (solution_id) solution_id
-            FROM dim_solution
-            LEFT JOIN (
-              SELECT solution_id, vulnerability_id, date_modified
-              FROM dim_vulnerability
-              LEFT JOIN dim_vulnerability_solution idvs USING (vulnerability_id)
-            ) ndvs USING (solution_id)
-            WHERE vulnerability_id IS NULL
-       ) dvs USING (solution_id)
-       LEFT JOIN (
-           SELECT DISTINCT (solution_id) solution_id,
-             array_agg(required_solution_id) as req_solutions
-           FROM dim_solution_prerequisite
-           GROUP BY solution_id
-       ) dsp USING (solution_id)
-       JOIN (
-           SELECT DISTINCT (solution_id) solution_id,
-             array_agg(superceding_solution_id) as super_solutions
-           FROM dim_solution_highest_supercedence
-           GROUP BY solution_id
-       ) dshs USING (solution_id)
-       ORDER BY solution_id"
-    end
-
-    def self.latest_scans(options={})
-      'SELECT ds.site_id, ds.name, ds.last_scan_id, dsc.finished
-        FROM dim_site ds
-        JOIN dim_scan dsc ON ds.last_scan_id = dsc.scan_id'
-    end
-  end
-end