nexpose_servicenow 0.7.3 → 0.8.0

data/lib/nexpose_servicenow/queries/queries_base.rb

@@ -1,25 +0,0 @@
-module NexposeServiceNow
-  class QueriesBase
-    def self.single_report?(query_name)
-      single_queries = %w(vulnerabilities vulnerability_category
-                          asset_groups latest_scans
-                          vulnerability_solutions vulnerability_references
-                          )
-      return single_queries.include? query_name.to_s
-    end
-
-    def self.csv_diff_required?(query_name)
-      diff_required = %w(asset_groups asset_group_memberships)
-      return (diff_required.include? query_name.to_s)
-    end
-
-    # TODO: Refactor this.
-    def self.query_keys(query_name)
-      if query_name.to_s == 'asset_groups'
-        [0]
-      else
-        [0,1]
-      end
-    end
-  end
-end

data/lib/nexpose_servicenow/queries/warehouse_queries.rb

@@ -1,341 +0,0 @@
-require_relative './queries_base'
-require_relative '../nx_logger'
-
-module NexposeServiceNow
-  class WarehouseQueries < QueriesBase
-    def self.latest_scans(options={})
-      'SELECT ds.site_id, ds.name, ds.last_scan_id, dsc.finished
-          FROM dim_site ds
-          JOIN dim_scan dsc ON ds.last_scan_id = dsc.scan_id'
-    end
-
-    def self.vulnerabilities(options={})
-      "SELECT
-          concat('R7_', vulnerability_id) AS ID,
-          cve.ref AS CVE,
-          cwe.ref AS CWE,
-          concat('Rapid7 Nexpose') AS Source,
-          to_char(date_published, 'yyyy-MM-dd hh:mm:ss') AS date_published,
-          to_char(date_modified, 'yyyy-MM-dd hh:mm:ss') AS Last_Modified,
-          dvc.category,
-          severity AS Severity_Rating,
-          severity_score AS Severity,
-          pci_status,
-          pci_adjusted_cvss_score AS PCI_Severity,
-          coalesce(NULLIF(CONCAT('\"',regexp_replace(title, '\"','''', 'g'),'\"'), '\"\"'), 'None') AS Summary,
-          coalesce(NULLIF(CONCAT('\"',regexp_replace(htmltotext(description), '\"','''', 'g'),'\"'), '\"\"'), 'None') AS Threat,
-          ROUND(risk_score :: NUMERIC, 2) AS Riskscore,
-          coalesce(cvss#{options[:cvss_v][:choice]}_vector,
-                   cvss#{options[:cvss_v][:fallback]}_vector) as cvss_vector,
-          ROUND(coalesce(cvss#{options[:cvss_v][:choice]}_impact_score::NUMERIC,
-                         cvss#{options[:cvss_v][:fallback]}_impact_score::NUMERIC),
-                2) AS Impact_Score,
-          ROUND(coalesce(cvss#{options[:cvss_v][:choice]}_exploit_score::NUMERIC,
-                         cvss#{options[:cvss_v][:fallback]}_exploit_score::NUMERIC),
-                2) AS Exploit_Score,
-          cvss_access_complexity AS Access_Complexity,
-          cvss_access_vector AS Access_Vector,
-          cvss_authentication AS Authentication,
-          ROUND(coalesce(cvss#{options[:cvss_v][:choice]}_score::NUMERIC,
-                         cvss#{options[:cvss_v][:fallback]}_score::NUMERIC),
-                2) as Vulnerability_Score,
-          cvss_integrity_impact AS Integrity_Impact,
-          cvss_confidentiality_impact AS Confidentiality_Impact,
-          cvss_availability_impact AS Availability_Impact,
-          (CASE
-           WHEN exploits > 0
-             THEN 'true'
-           ELSE 'false'
-           END) AS Exploitability,
-          (CASE
-           WHEN malware_kits > 0
-             THEN 'true'
-           ELSE 'false'
-           END) AS Malware_Kits,
-          NULLIF(CONCAT('\"', array_to_string(sol.solutions, ',', ''), '\"'), '\"\"') AS Solution
-        FROM
-          dim_vulnerability
-          LEFT OUTER JOIN
-          (SELECT DISTINCT ON (vulnerability_id)
-             vulnerability_id,
-             dvr.reference AS ref
-           FROM dim_vulnerability_reference dvr
-           WHERE source = 'CWE'
-           GROUP BY dvr.vulnerability_id, dvr.reference
-          ) cwe USING (vulnerability_id)
-          LEFT OUTER JOIN
-          (SELECT DISTINCT ON (vulnerability_id)
-             vulnerability_id,
-             dvr.reference AS ref
-           FROM dim_vulnerability_reference dvr
-           WHERE source = 'CVE'
-           GROUP BY dvr.vulnerability_id, dvr.reference
-          ) cve USING (vulnerability_id)
-          LEFT OUTER JOIN (SELECT DISTINCT ON (dvc.vulnerability_id)
-                             dvc.vulnerability_id,
-                             dvc.category_name AS category
-                           FROM dim_vulnerability_category dvc
-                           GROUP BY dvc.vulnerability_id, dvc.category_name) dvc USING (vulnerability_id)
-          LEFT OUTER JOIN (SELECT
-                             dvr.vulnerability_id,
-                             string_agg(dvr.source || ': ' || dvr.reference, '|') AS references
-                           FROM dim_vulnerability_reference dvr
-                           GROUP BY dvr.vulnerability_id) ref USING (vulnerability_id)
-          LEFT OUTER JOIN (SELECT
-                             vulnerability_id,
-                             array_agg(solution_id) AS solutions
-                           FROM dim_vulnerability_solution
-                           GROUP BY vulnerability_id) sol USING (vulnerability_id)
-        WHERE date_modified >= '#{options[:vuln_query_date]}'"
-    end
-
-    def self.vulnerability_references(options={})
-      "SELECT concat('R7_', vulnerability_id) as ID,
-              Source,
-              Reference
-        FROM (SELECT vulnerability_id
-              FROM dim_vulnerability
-              WHERE date_modified >= '#{options[:vuln_query_date]}') dv
-        JOIN (SELECT vulnerability_id, Source, Reference
-            FROM dim_vulnerability_reference) dvr USING (vulnerability_id)"
-    end
-
-    def self.vulnerability_category(options={})
-      "SELECT concat('R7_', vulnerability_id) as ID, dvc.Category
-        FROM dim_vulnerability
-        LEFT OUTER JOIN
-                (SELECT vulnerability_id, category_name as Category
-                FROM dim_vulnerability_category dvc) dvc USING (vulnerability_id)
-        WHERE date_modified >= '#{options[:vuln_query_date]}'"
-    end
-
-    def self.assets(options={})
-      last_import_date = options[:delta]
-      site_id = options[:site_id]
-
-      "WITH scan_found AS (
-          SELECT DISTINCT (asset_id)
-            asset_id,
-            MIN(scan_id) AS scan_found
-          FROM fact_asset_event
-          WHERE type = 'SCAN'
-          GROUP BY asset_id
-        ), new_assets AS (
-          SELECT
-            sf.asset_id
-          FROM dim_scan ds
-            JOIN scan_found sf ON (ds.scan_id = sf.scan_found)
-          WHERE ds.finished > '#{last_import_date}'
-          AND ds.site_id = #{site_id}
-        )
-        SELECT
-          coalesce(host_name, CAST(dim_asset.asset_id AS TEXT)) AS Name,
-          dim_asset.ip_address,
-          dim_asset.mac_address,
-          concat('Rapid7 Nexpose') AS Discovery_Source,
-          CAST(CASE
-               WHEN dim_asset.host_type = 'Virtual Machine' OR dim_asset.host_type = 'Hypervisor'
-                 THEN 1
-               ELSE 0
-               END AS BIT) AS Is_Virtual,
-          CONCAT('\"', dim_asset.os_description, '\"') AS Operating_System,
-          dim_asset.last_assessed_for_vulnerabilities AS Most_Recent_Discovery,
-          dim_asset.asset_id AS Nexpose_ID,
-          fact_asset.pci_status,
-          dim_asset.credential_status
-        FROM dim_asset
-          JOIN fact_asset USING (asset_id)
-        WHERE dim_asset.asset_id IN (
-          SELECT asset_id
-          FROM new_assets
-        )"
-    end
-
-    def self.generate_vulnerability_filter(cves, cvss_range, cvss_strings)
-      return '' if cves.to_a.empty? && cvss_range.to_a.empty?
-      vuln_filter = ''
-      vuln_filter += cves.map { |c| "reference='#{c}'" }.join(' OR ') unless cves.nil?
-
-      cvss_min = cvss_range.first || '0'
-      cvss_max = cvss_range.last || '10'
-
-      cvss_score = "(coalesce(cvss#{cvss_strings[:choice]}_score,
-                      cvss#{cvss_strings[:fallback]}_score))"
-
-      unless cvss_min.to_s == '0' && cvss_max.to_s == '10'
-        vuln_filter += ' AND ' unless vuln_filter.empty?
-        vuln_filter += "#{cvss_score} >= #{cvss_min} AND #{cvss_score} <= #{cvss_max}"
-      end
-      return '' if vuln_filter.empty?
-
-      "AND favi.vulnerability_id IN (
-          SELECT dvr.vulnerability_id
-          FROM dim_vulnerability_reference dvr
-            JOIN dim_vulnerability dv ON dv.vulnerability_id = dvr.vulnerability_id
-          WHERE dvr.vulnerability_id IN (SELECT av.vulnerability_id FROM asset_vulns av)
-          AND #{vuln_filter})"
-    end
-
-    def self.generate_date_filter(dates)
-      return '' if dates.to_a.empty?
-      filters = []
-      filters << "first_found > '#{dates.first}'" unless dates.first.nil?
-      filters << "first_found < '#{dates.last}'" unless dates.last.nil?
-      filters.empty? ? '' : " AND #{filters.join(' AND ')}"
-    end
-
-    def self.vulnerable_new_items(options={})
-      site_id = options[:site_id]
-      last_import_date = options[:delta]
-      vuln_filter = self.generate_vulnerability_filter(
-          options[:filters][:cve], options[:filters][:cvss], options[:cvss_v]
-      )
-      date_filters = self.generate_date_filter(options[:filters][:date])
-
-      "WITH asset_vulns AS (
-          SELECT DISTINCT ON (favf.asset_id, favf.vulnerability_id)
-            favf.asset_id,
-            favf.vulnerability_id,
-            favf.date AS first_found,
-            favf.vulnerability_instances,
-            ip_address,
-            (SELECT max(date)
-             FROM fact_asset_event fae
-             WHERE type = 'SCAN'
-                   AND fae.asset_id = favf.asset_id
-             GROUP BY fae.asset_id) AS last_found
-          FROM fact_asset_vulnerability_finding favf
-            JOIN dim_site_asset dsa ON favf.asset_id = dsa.asset_id AND dsa.site_id = #{site_id}
-            JOIN dim_asset da ON favf.asset_id = da.asset_id
-          WHERE date > '#{last_import_date}'
-      ), valid_vulns AS (
-          SELECT DISTINCT ON (favi.asset_id, favi.vulnerability_id)
-            favi.asset_id,
-            favi.vulnerability_id,
-            array_agg(DISTINCT favi.port)                          AS ports,
-            array_agg(DISTINCT davs.solution_id)                   AS solutions,
-            favi.protocol,
-            regexp_replace(htmltotext(favi.proof), '\"','''', 'g') AS proof,
-            favi.status
-          FROM fact_asset_vulnerability_instance favi
-            LEFT JOIN dim_asset_vulnerability_finding_solution davs ON
-            davs.asset_id = favi.asset_id AND davs.vulnerability_id = favi.vulnerability_id
-          WHERE (favi.vulnerability_id, favi.asset_id) IN (SELECT av.vulnerability_id, av.asset_id FROM asset_vulns av)
-          #{vuln_filter}
-          GROUP BY favi.asset_id, favi.vulnerability_id, protocol, proof, status
-      )
-      SELECT
-        CAST(nv.asset_id AS TEXT)                                      AS Configuration_Item,
-        'true'                                                         AS Active,
-        concat('R7_', nv.vulnerability_id)                             AS Vulnerability,
-        nv.first_found                                                 AS First_Found,
-        nv.last_found                                                  AS Last_Found,
-        nv.vulnerability_instances                                     AS Times_Found,
-        nv.ip_address                                                  AS IP_Address,
-        coalesce(NULLIF(CONCAT('\"', vv.proof, '\"'), '\"\"'), 'None') AS Proof,
-        vv.status                                                      AS Status,
-        array_to_string(vv.ports, ' ', '')                             AS Ports,
-        coalesce(vv.protocol, 'None')                                  AS Protocol,
-        NULLIF(CONCAT('\"', array_to_string(vv.solutions, ',', ''), '\"'), '\"\"') AS Solutions
-      FROM asset_vulns nv
-        JOIN valid_vulns vv ON vv.asset_id = nv.asset_id AND vv.vulnerability_id = nv.vulnerability_id
-      #{date_filters}
-      ORDER BY nv.asset_id, Vulnerability"
-    end
-
-    def self.vulnerable_old_items(options={})
-      site_id = options[:site_id]
-      last_import_date = options[:delta]
-      vuln_filter = self.generate_vulnerability_filter(
-          options[:filters][:cve], options[:filters][:cvss], options[:cvss_v]
-      )
-      date_filters = self.generate_date_filter(options[:filters][:date])
-
-      "WITH site_assets AS (
-          SELECT asset_id
-          FROM dim_site_asset
-          WHERE site_id = #{site_id}
-      ), asset_vulns AS (
-          -- Asset vulns at last import
-          SELECT asset_id, vulnerability_id, date AS first_found
-          FROM fact_asset_vulnerability_finding_date
-          WHERE asset_id IN (SELECT asset_id FROM site_assets)
-          AND DAY = (SELECT periodbefore('#{last_import_date}'))
-      ), current_vulns AS (
-          SELECT asset_id, vulnerability_id
-          FROM fact_asset_vulnerability_finding
-          WHERE asset_id IN (SELECT asset_id FROM site_assets)
-      )
-      SELECT
-          CAST(da.asset_id AS TEXT)            AS Configuration_Item,
-          'false'                              AS Active,
-          concat('R7_', favi.vulnerability_id) AS Vulnerability,
-          da.ip_address                        AS IP_Address
-      FROM dim_asset da
-          -- Name this table favi to use the vulnerability ID with the vuln filter.
-          JOIN asset_vulns favi ON (da.asset_id = favi.asset_id)
-          LEFT JOIN current_vulns cv ON (cv.asset_id = da.asset_id) AND (cv.vulnerability_id = favi.vulnerability_id)
-      WHERE da.asset_id IN (SELECT asset_id FROM site_assets)
-      AND cv.vulnerability_id IS NULL
-      #{vuln_filter}
-      #{date_filters}
-      ORDER BY da.asset_id, Vulnerability"
-    end
-
-    def self.vulnerability_solutions(options={})
-      "SELECT DISTINCT (solution_id)
-         solution_id,
-         nexpose_id,
-         coalesce(NULLIF(CONCAT('\"',regexp_replace(htmltotext(fix), '\"','''', 'g'),'\"'), '\"\"'), 'None')             as fix,
-         estimate,
-         coalesce(NULLIF(CONCAT('\"',regexp_replace(htmltotext(summary), '\"','''', 'g'),'\"'), '\"\"'), 'None')         as summary,
-         solution_type,
-         coalesce(NULLIF(CONCAT('\"',regexp_replace(htmltotext(applies_to), '\"','''', 'g'),'\"'), '\"\"'), 'None')      as applies_to,
-         coalesce(NULLIF(CONCAT('\"', url, '\"'), '\"\"'), 'None')                                                       as url,
-         coalesce(NULLIF(CONCAT('\"',regexp_replace(htmltotext(additional_data), '\"','''', 'g'),'\"'), '\"\"'), 'None') as additional_data,
-         CONCAT('\"', array_to_string(req_solutions, ',', ''), '\"')                                                     as required_solutions,
-         CONCAT('\"', array_to_string(super_solutions, ',', ''), '\"')                                                   as superceding_solutions
-       FROM dim_solution
-       RIGHT OUTER JOIN (
-           SELECT DISTINCT (solution_id) solution_id
-            FROM (
-              SELECT solution_id, vulnerability_id, date_modified
-              FROM dim_vulnerability
-              LEFT JOIN dim_vulnerability_solution idvs USING (vulnerability_id)
-            ) dvs
-            WHERE date_modified >= '#{options[:vuln_query_date]}'
-            UNION
-            SELECT DISTINCT (solution_id) solution_id
-            FROM dim_solution
-            LEFT JOIN (
-              SELECT solution_id, vulnerability_id, date_modified
-              FROM dim_vulnerability
-              LEFT JOIN dim_vulnerability_solution idvs USING (vulnerability_id)
-            ) ndvs USING (solution_id)
-            WHERE vulnerability_id IS NULL
-       ) dvs USING (solution_id)
-       LEFT JOIN (
-           SELECT DISTINCT (solution_id) solution_id,
-             array_agg(required_solution_id) as req_solutions
-           FROM dim_solution_prerequisite
-           GROUP BY solution_id
-       ) dsp USING (solution_id)
-       JOIN (
-           SELECT DISTINCT (solution_id) solution_id,
-             array_agg(superceding_solution_id) as super_solutions
-           FROM dim_solution_highest_supercedence
-           GROUP BY solution_id
-       ) dshs USING (solution_id)
-       ORDER BY solution_id"
-    end
-
-    def self.method_missing(m, *args, &block)
-      msg = "Method #{m} is not yet available for data warehouse mode."
-
-      log = NexposeServiceNow::NxLogger.instance
-      log.log_message(msg)
-      puts "#{msg} Exiting..."
-      exit 0
-    end
-  end
-end