nexpose_servicenow 0.4.18 → 0.4.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c325a43d283e142343eb252319480592167450ab
-  data.tar.gz: a5177c3ac6503e29cb039a159dcf0b8c508f1bf8
+  metadata.gz: c0e5533db3d8c9507f29fb3d72c826dd3dabbff5
+  data.tar.gz: 546ec9e53c92c594e403c09aa04b6ea7a36ee24c
 SHA512:
-  metadata.gz: a47fb845d6acd135d3411dee4adb77a07ecc4a81bd48387f3f9bbd419d2d152552cfc0cfc42235210ebc512a87f8b2b97cf388382f1518de7de86cc3e5fe49bd
-  data.tar.gz: 7812e7398e897d8ba6d21c29164f5d430f9c63aafd1ffb6c8d68c89aca3c1b571842a20c48341f7b0afac3569449b1a172b35fb1544422349e36102369f90b87
+  metadata.gz: e7909b1301dfb21cb8a359481f3a80712385aac95cce0272f7177397d45895c23e6d9d0c7061c0b574b20671b590597b4895a0d84c22beadd1e256c17d329203
+  data.tar.gz: 527f47260c70edee76bb7c2eaf010fe09206ca9cd44ebe6b21fb895773f505c8ebf1bc6e36c2e47921432f1664110dd61d96acea5a01f06ac9fa8877e3950a1a

data/lib/nexpose_servicenow.rb

@@ -22,8 +22,14 @@ module NexposeServiceNow
       censored_options[:nexpose_password] = "*****"
       log.log_message("Options: #{censored_options}")
 
+      if options[:mode].to_s == ""
+        log.log_message("Script was called without mode.")
+        puts "No mode selected. Use -h to see command line options."
+        exit -1
+      end
+
       if options[:nexpose_ids].first.to_s == "0"
-        log.log_message('Retrieving array of all site IDs')
+        log.log_error_message('Retrieving array of all site IDs')
         options[:nexpose_ids] = get_nexpose_helper(options).all_sites.sort
       end             
 
@@ -79,6 +85,13 @@ module NexposeServiceNow
         return if report_details.all? { |f| File.exists?(f[:report_name]) } 
       end
 
+      credentials = %i{nexpose_username nexpose_password}
+      if credentials.any? { |cred| options[cred].to_s == "" }
+        log = NexposeServiceNow::NxLogger.instance
+        log.log_error_message "Nexpose credentials necessary but not supplied."
+        exit -1
+      end
+
       #Filter it down to sites which actively need queried
       sites_to_scan = filter_sites(options)
       nexpose_helper = get_nexpose_helper(options)

data/lib/nexpose_servicenow/chunker.rb

@@ -26,10 +26,9 @@ module NexposeServiceNow
     end
 
     def preprocess(nexpose_ids=nil)
-      @log.log_message("Breaking file #{@file_path} down into chunks.")
-
       all_chunks = []
       @report_details.each do |report|
+        @log.log_message("Breaking file #{report[:report_name]} down into chunks.")
         chunks = process_file(report[:report_name], report[:id])
         all_chunks.concat chunks
       end 

data/lib/nexpose_servicenow/historical_data.rb

@@ -127,7 +127,12 @@ module NexposeServiceNow
       #merge changes in from remote_csv
       remote_csv.each do |row|
         updated_row = updated_csv.find { |r| r['site_id'] == row['site_id'] }
-        updated_row['last_scan_id'] = row['last_scan_id']
+        if updated_row.nil?
+          row.delete 'finished'
+          updated_csv << row    
+        else
+          updated_row['last_scan_id'] = row['last_scan_id'] 
+        end
       end
 
       save_last_scan_data(updated_csv) 

data/lib/nexpose_servicenow/nexpose_helper.rb

@@ -9,6 +9,9 @@ module NexposeServiceNow
       @log = NexposeServiceNow::NxLogger.instance
     	@url = url
     	@port = port
+      @username = username
+      @password = password
+
     	@nsc = connect(username, password)
 
       @timeout = 7200
@@ -131,9 +134,17 @@ module NexposeServiceNow
 		  File.open(local_file_name, 'wb') do |f| 
         f.write(@nsc.download(report_details.uri))
       end
+      
+      begin
+        # Refresh the connection
+        @nsc = connect(@username, @password)
+
+		    #Got the report, cleanup server-side
+		    @nsc.delete_report_config(report_id)
+      rescue
+        @log.log_error_message "Error deleting report"
+      end
 
-		  #Got the report, cleanup server-side
-		  @nsc.delete_report_config(report_id)
       local_file_name
     end
 

data/lib/nexpose_servicenow/queries.rb

@@ -180,35 +180,34 @@ module NexposeServiceNow
     end
 
     def self.vulnerable_new_items(options={})
-      "SELECT 
+      "SELECT
       coalesce(subq.host_name, CAST(subq.asset_id as text)) Configuration_Item,
       TRUE as Active,
       concat('R7_', subq.vulnerability_id) as Vulnerability,
-      fasv.first_discovered as First_Found,
-      fasv.most_recently_discovered as Last_Found,
+      fasva.first_discovered as First_Found,
+      fasva.most_recently_discovered as Last_Found,
       subq.vulnerability_instances as Times_Found,
       subq.ip_address as IP_Address,
       favi.port as Port,
-      dp.name as Protocol
+      coalesce(NULLIF(favi.name,''), 'None') as Protocol
 
       FROM (
-      SELECT fasv.asset_id, fasv.vulnerability_id, vulnerability_instances, s.current_scan, s.host_name, s.ip_address, baselineComparison(fasv.scan_id, s.current_scan) as comparison 
+             SELECT fasv.asset_id, fasv.vulnerability_id, vulnerability_instances, 
+                     MIN(fasv.scan_id) as first_found, MAX(fasv.scan_id) as latest_found, 
+                     s.current_scan, s.host_name, s.ip_address
               FROM fact_asset_scan_vulnerability_finding fasv
               JOIN (
-                  SELECT asset_id, host_name, ip_address, previousScan(asset_id) AS baseline_scan, lastScan(asset_id) AS current_scan FROM dim_asset
-               ) s ON s.asset_id = fasv.asset_id AND (fasv.scan_id >= #{options[:last_scan_id]} OR fasv.scan_id = s.current_scan)
+                  SELECT asset_id, host_name, ip_address, lastScan(asset_id) AS current_scan FROM dim_asset
+               ) s ON s.asset_id = fasv.asset_id
                GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan, s.host_name, s.ip_address, vulnerability_instances
-               HAVING baselineComparison(fasv.scan_id, s.current_scan) = 'New'
+               HAVING MIN(fasv.scan_id) > #{options[:last_scan_id]} AND MAX(fasv.scan_id)=current_scan
               ) subq
-      JOIN 
-              fact_asset_vulnerability_instance favi ON favi.asset_id = subq.asset_id AND favi.scan_id = subq.current_scan AND favi.vulnerability_id = subq.vulnerability_id
-            JOIN 
-              fact_asset_vulnerability_age fasv ON fasv.asset_id = subq.asset_id AND fasv.vulnerability_id = subq.vulnerability_id
-            JOIN
-              dim_asset da ON subq.asset_id = da.asset_id
-            JOIN 
-              dim_protocol dp ON dp.protocol_id = favi.protocol_id
-      ORDER BY fasv.asset_id, vulnerability"
+      JOIN (select asset_id, vulnerability_id, first_discovered, most_recently_discovered from fact_asset_vulnerability_age) fasva ON fasva.asset_id = subq.asset_id AND fasva.vulnerability_id = subq.vulnerability_id
+
+      JOIN (select DISTINCT on(asset_id, vulnerability_id) asset_id, scan_id, vulnerability_id, port, dp.name
+                      from fact_asset_vulnerability_instance
+                      inner join dim_protocol dp USING (protocol_id)) favi ON favi.asset_id = subq.asset_id AND favi.scan_id = subq.current_scan AND favi.vulnerability_id = subq.vulnerability_id
+      ORDER BY fasva.asset_id, vulnerability"
     end
 
     def self.vulnerable_old_items(options={})
@@ -218,14 +217,15 @@ module NexposeServiceNow
       concat('R7_', subq.vulnerability_id) as Vulnerability,
       da.ip_address as IP_Address
       FROM (
-              SELECT fasv.asset_id, fasv.vulnerability_id, s.current_scan, baselineComparison(fasv.scan_id, s.current_scan) as comparison 
+             SELECT fasv.asset_id, fasv.vulnerability_id, MAX(fasv.scan_id) as latest_found, 
+                     s.current_scan, s.host_name, s.ip_address
               FROM fact_asset_scan_vulnerability_finding fasv
               JOIN (
-                  select asset_id, lastScan(asset_id) AS current_scan from fact_asset_vulnerability_instance WHERE scan_id = lastScan(asset_id)
-               ) s ON s.asset_id = fasv.asset_id AND (fasv.scan_id >= #{options[:last_scan_id]} OR fasv.scan_id = s.current_scan)
-               GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan
-               HAVING baselineComparison(fasv.scan_id, s.current_scan) = 'Old'
-           ) subq
+                  SELECT asset_id, host_name, ip_address, lastScan(asset_id) AS current_scan FROM dim_asset
+               ) s ON s.asset_id = fasv.asset_id
+               GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan, s.host_name, s.ip_address, vulnerability_instances
+               HAVING MAX(fasv.scan_id) < current_scan AND MAX(fasv.scan_id) >= #{options[:last_scan_id]} 
+              ) subq
       JOIN dim_asset da ON subq.asset_id = da.asset_id
       ORDER BY da.ip_address"
     end

data/lib/nexpose_servicenow/version.rb

@@ -1,5 +1,5 @@
 module NexposeServiceNow
-  VERSION = "0.4.18"
+  VERSION = "0.4.22"
   VENDOR = "ServiceNow"
   PRODUCT = "CMDB"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nexpose_servicenow
 version: !ruby/object:Gem::Version
-  version: 0.4.18
+  version: 0.4.22
 platform: ruby
 authors:
 - David Valente
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-16 00:00:00.000000000 Z
+date: 2016-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -74,7 +74,6 @@ files:
 - lib/nexpose_servicenow/nexpose_helper.rb
 - lib/nexpose_servicenow/nx_logger.rb
 - lib/nexpose_servicenow/queries.rb
-- lib/nexpose_servicenow/queries_original.rb
 - lib/nexpose_servicenow/version.rb
 - nexpose_servicenow.gemspec
 homepage: http://www.rapid7.com
@@ -98,7 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Gem for Nexpose-ServiceNow integration.

data/lib/nexpose_servicenow/queries_original.rb

@@ -1,162 +0,0 @@
-module NexposeServiceNow
-	class Queries
-    def self.cmdb_ci_outofband_device(nexpose_url)
-      "SELECT coalesce(host_name, CAST(dim_asset.asset_id as text)) as Name,
-        host_name as Aliases,
-        ip_address as IP_Address,
-        concat('https://#{nexpose_url}/asset.jsp?devid=', dim_asset.asset_id) as URL,
-        dim_host_type.description as Type,
-        dim_operating_system.description as Product_Version,
-        fa.scan_finished as Most_Recent_Discovery,
-        fa.vulnerabilities as Vulnerabilities,
-        fa.critical_vulnerabilities as Critical_Vulnerabilities,
-        fa.severe_vulnerabilities as Severe_Vulnerabilities,
-        fa.moderate_vulnerabilities as Moderate_Vulnerabilities,
-        fa.malware_kits as Malware_kits,
-        fa.exploits as Exploits,
-        fa.vulnerabilities_with_malware_kit as Vulnerabilities_With_Malware_Kit,
-        fa.vulnerabilities_with_exploit as Vulnerabilities_With_Exploit,
-        fa.vulnerability_instances as Vulnerability_Instances,
-        dim_asset.asset_id as Nexpose_ID,
-        fa.riskscore as Risk_Score,
-        ga.group_accounts as Group_Accounts,
-        ag.asset_groups as Asset_Groups,
-        serv.services as Services,
-        softw.software as Software,
-        use.user_accounts as User_Accounts,
-        site.sites as Sites,
-        tag.tags as Tags,
-        fa.pci_status as PCI_Status
-
-
-        FROM dim_asset
-        LEFT OUTER JOIN dim_host_type on dim_asset.host_type_id = dim_host_type.host_type_id
-        LEFT OUTER JOIN dim_operating_system on dim_asset.operating_system_id = dim_operating_system.operating_system_id
-        
-        JOIN fact_asset fa USING (asset_id)
-
-        LEFT OUTER JOIN (SELECT daga.asset_id, string_agg(daga.name, '|') as Group_Accounts
-          FROM dim_asset_group_account daga
-          GROUP BY daga.asset_id) ga USING (asset_id)
-
-
-        LEFT OUTER JOIN (SELECT daga.asset_id, string_agg(dag.name, '|') as Asset_Groups
-          FROM dim_asset_group_asset daga
-          JOIN dim_asset_group dag on daga.asset_group_id = dag.asset_group_id
-          GROUP BY daga.asset_id) ag USING (asset_id)
-
-
-        LEFT OUTER JOIN (SELECT das.asset_id, string_agg(ds.name, '|') as Services
-          FROM dim_asset_service das
-          JOIN dim_service ds on das.service_id = ds.service_id
-          GROUP BY das.asset_id) serv USING (asset_id)
-
-        LEFT OUTER JOIN (SELECT dauc.asset_id, string_agg(dauc.name, '|') as User_Accounts
-          FROM dim_asset_user_account dauc
-          GROUP BY dauc.asset_id) use USING (asset_id)
-
-        LEFT OUTER JOIN (SELECT dsa.asset_id, string_agg(ds.name, '|') as Sites
-          FROM dim_site_asset dsa
-          JOIN dim_site ds on dsa.site_id = ds.site_id
-          GROUP BY dsa.asset_id) site USING (asset_id)
-
-        LEFT OUTER JOIN (SELECT das.asset_id, string_agg(ds.name, '|') as Software
-          FROM dim_asset_software das
-          JOIN dim_software ds on das.software_id = ds.software_id
-          GROUP BY das.asset_id) softw USING (asset_id)
-
-        LEFT OUTER JOIN (SELECT dta.asset_id, string_agg(dt.tag_name, '|') as Tags
-          FROM dim_tag_asset dta
-          JOIN dim_tag dt on dta.tag_id = dt.tag_id
-          GROUP BY dta.asset_id) tag USING (asset_id)
-
-        GROUP BY dim_asset.host_name, dim_asset.asset_id, dim_asset.ip_address, fa.pci_status, dim_host_type.description, 
-        dim_operating_system.description, fa.scan_finished, fa.vulnerabilities, fa.critical_vulnerabilities, fa.severe_vulnerabilities,
-        fa.moderate_vulnerabilities, fa.malware_kits, fa.exploits, fa.vulnerabilities_with_malware_kit, fa.vulnerabilities_with_exploit,
-        fa.vulnerability_instances, fa.riskscore, ga.group_accounts, softw.software, ag.asset_groups, serv.services, use.user_accounts, site.sites, tag.tags"
-    end
-    
-    def self.sn_vul_vulnerable_item(options={})
-      "SELECT 
-        asset_id as Configuration_Item,
-        concat('R7_', vulnerability_id) as Vulnerability,
-        fasv.first_discovered as First_Found,
-        fasv.most_recently_discovered as Last_Found,
-        fact_asset_vulnerability_finding.vulnerability_instances as Times_Found,
-        dim_asset.ip_address as IP_Address,
-        port as Port,
-        dim_protocol.name as Protocol
-
-      FROM 
-        fact_asset_vulnerability_instance
-      JOIN 
-        fact_asset_vulnerability_finding USING (asset_id, vulnerability_id)
-      JOIN 
-        fact_asset_vulnerability_age fasv USING (asset_id, vulnerability_id)
-      JOIN
-        dim_asset USING (asset_id)
-      JOIN 
-        dim_protocol USING (protocol_id)"
-    end
-
-    def self.sn_vul_third_party_entry(options={})
-      "SELECT
-        concat('R7_', vulnerability_id) as ID, 
-        cve.ref as CVE,
-        cwe.ref as CWE,
-        concat('Rapid7 Nexpose') as Source,
-        date_published,
-        date_modified as Last_Modified,
-        dvc.categories,
-        severity_score as Severity,
-        title as Summary,
-        description as Threat,
-        ROUND(riskscore::numeric, 2) as Riskscore,
-        cvss_vector,
-        ROUND(cvss_score::numeric, 2) as CVSS_Score,
-        exploits,
-        ref.references,
-        sol.solutions
-
-      FROM
-        dim_vulnerabilityYep
-
-      LEFT OUTER JOIN
-      (SELECT 
-         vulnerability_id, 
-         string_agg(dvr.reference, '|') as ref
-       FROM dim_vulnerability_reference dvr
-       WHERE source='CWE'
-       GROUP BY dvr.vulnerability_id
-       ) cwe USING (vulnerability_id)
-
-      LEFT OUTER JOIN
-      (SELECT 
-       vulnerability_id,
-       string_agg(dvr.reference, '|') as ref
-       FROM dim_vulnerability_reference dvr
-       WHERE source='CVE'
-       GROUP BY dvr.vulnerability_id
-      ) cve USING (vulnerability_id)
-
-      LEFT OUTER JOIN(SELECT dvc.vulnerability_id, string_agg(dvc.category_name, '|') as categories
-        FROM dim_vulnerability_category dvc
-        GROUP BY dvc.vulnerability_id) dvc USING (vulnerability_id)
-
-      LEFT OUTER JOIN(SELECT dvr.vulnerability_id, string_agg(dvr.source || ': ' || dvr.reference, '|') as references
-        FROM dim_vulnerability_reference dvr
-        GROUP BY dvr.vulnerability_id) ref USING (vulnerability_id)
-
-      LEFT OUTER JOIN(SELECT vulnerability_id, 
-                        string_agg(concat('Fix: ' || fix, 
-                       'Solution type: ' || solution_type, 
-                       'URL: ' || url,
-                       'Estimate: ' || estimate,
-                       'Applies To: ' || applies_to,
-                       'Additional Data: ' || additional_data), '|') as solutions
-        FROM dim_solution
-        JOIN dim_vulnerability_solution USING (solution_id)
-        GROUP BY vulnerability_id) sol USING (vulnerability_id)"
-    end
-  end
-end