nexpose_servicenow 0.7.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +51 -15
- data/bin/nexpose_servicenow +10 -2
- data/lib/nexpose_servicenow.rb +1 -174
- data/lib/nexpose_servicenow/version.rb +14 -1
- data/nexpose_servicenow.gemspec +2 -4
- metadata +13 -59
- data/lib/nexpose_servicenow/arg_parser.rb +0 -283
- data/lib/nexpose_servicenow/chunker.rb +0 -109
- data/lib/nexpose_servicenow/csv_compare.rb +0 -177
- data/lib/nexpose_servicenow/helpers/connection_helper.rb +0 -84
- data/lib/nexpose_servicenow/helpers/data_warehouse_helper.rb +0 -140
- data/lib/nexpose_servicenow/helpers/nexpose_console_helper.rb +0 -164
- data/lib/nexpose_servicenow/historical_data.rb +0 -102
- data/lib/nexpose_servicenow/nx_logger.rb +0 -166
- data/lib/nexpose_servicenow/queries/nexpose_queries.rb +0 -459
- data/lib/nexpose_servicenow/queries/queries_base.rb +0 -25
- data/lib/nexpose_servicenow/queries/warehouse_queries.rb +0 -341
@@ -1,25 +0,0 @@
|
|
1
|
-
module NexposeServiceNow
|
2
|
-
class QueriesBase
|
3
|
-
def self.single_report?(query_name)
|
4
|
-
single_queries = %w(vulnerabilities vulnerability_category
|
5
|
-
asset_groups latest_scans
|
6
|
-
vulnerability_solutions vulnerability_references
|
7
|
-
)
|
8
|
-
return single_queries.include? query_name.to_s
|
9
|
-
end
|
10
|
-
|
11
|
-
def self.csv_diff_required?(query_name)
|
12
|
-
diff_required = %w(asset_groups asset_group_memberships)
|
13
|
-
return (diff_required.include? query_name.to_s)
|
14
|
-
end
|
15
|
-
|
16
|
-
# TODO: Refactor this.
|
17
|
-
def self.query_keys(query_name)
|
18
|
-
if query_name.to_s == 'asset_groups'
|
19
|
-
[0]
|
20
|
-
else
|
21
|
-
[0,1]
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
@@ -1,341 +0,0 @@
|
|
1
|
-
require_relative './queries_base'
|
2
|
-
require_relative '../nx_logger'
|
3
|
-
|
4
|
-
module NexposeServiceNow
|
5
|
-
class WarehouseQueries < QueriesBase
|
6
|
-
def self.latest_scans(options={})
|
7
|
-
'SELECT ds.site_id, ds.name, ds.last_scan_id, dsc.finished
|
8
|
-
FROM dim_site ds
|
9
|
-
JOIN dim_scan dsc ON ds.last_scan_id = dsc.scan_id'
|
10
|
-
end
|
11
|
-
|
12
|
-
def self.vulnerabilities(options={})
|
13
|
-
"SELECT
|
14
|
-
concat('R7_', vulnerability_id) AS ID,
|
15
|
-
cve.ref AS CVE,
|
16
|
-
cwe.ref AS CWE,
|
17
|
-
concat('Rapid7 Nexpose') AS Source,
|
18
|
-
to_char(date_published, 'yyyy-MM-dd hh:mm:ss') AS date_published,
|
19
|
-
to_char(date_modified, 'yyyy-MM-dd hh:mm:ss') AS Last_Modified,
|
20
|
-
dvc.category,
|
21
|
-
severity AS Severity_Rating,
|
22
|
-
severity_score AS Severity,
|
23
|
-
pci_status,
|
24
|
-
pci_adjusted_cvss_score AS PCI_Severity,
|
25
|
-
coalesce(NULLIF(CONCAT('\"',regexp_replace(title, '\"','''', 'g'),'\"'), '\"\"'), 'None') AS Summary,
|
26
|
-
coalesce(NULLIF(CONCAT('\"',regexp_replace(htmltotext(description), '\"','''', 'g'),'\"'), '\"\"'), 'None') AS Threat,
|
27
|
-
ROUND(risk_score :: NUMERIC, 2) AS Riskscore,
|
28
|
-
coalesce(cvss#{options[:cvss_v][:choice]}_vector,
|
29
|
-
cvss#{options[:cvss_v][:fallback]}_vector) as cvss_vector,
|
30
|
-
ROUND(coalesce(cvss#{options[:cvss_v][:choice]}_impact_score::NUMERIC,
|
31
|
-
cvss#{options[:cvss_v][:fallback]}_impact_score::NUMERIC),
|
32
|
-
2) AS Impact_Score,
|
33
|
-
ROUND(coalesce(cvss#{options[:cvss_v][:choice]}_exploit_score::NUMERIC,
|
34
|
-
cvss#{options[:cvss_v][:fallback]}_exploit_score::NUMERIC),
|
35
|
-
2) AS Exploit_Score,
|
36
|
-
cvss_access_complexity AS Access_Complexity,
|
37
|
-
cvss_access_vector AS Access_Vector,
|
38
|
-
cvss_authentication AS Authentication,
|
39
|
-
ROUND(coalesce(cvss#{options[:cvss_v][:choice]}_score::NUMERIC,
|
40
|
-
cvss#{options[:cvss_v][:fallback]}_score::NUMERIC),
|
41
|
-
2) as Vulnerability_Score,
|
42
|
-
cvss_integrity_impact AS Integrity_Impact,
|
43
|
-
cvss_confidentiality_impact AS Confidentiality_Impact,
|
44
|
-
cvss_availability_impact AS Availability_Impact,
|
45
|
-
(CASE
|
46
|
-
WHEN exploits > 0
|
47
|
-
THEN 'true'
|
48
|
-
ELSE 'false'
|
49
|
-
END) AS Exploitability,
|
50
|
-
(CASE
|
51
|
-
WHEN malware_kits > 0
|
52
|
-
THEN 'true'
|
53
|
-
ELSE 'false'
|
54
|
-
END) AS Malware_Kits,
|
55
|
-
NULLIF(CONCAT('\"', array_to_string(sol.solutions, ',', ''), '\"'), '\"\"') AS Solution
|
56
|
-
FROM
|
57
|
-
dim_vulnerability
|
58
|
-
LEFT OUTER JOIN
|
59
|
-
(SELECT DISTINCT ON (vulnerability_id)
|
60
|
-
vulnerability_id,
|
61
|
-
dvr.reference AS ref
|
62
|
-
FROM dim_vulnerability_reference dvr
|
63
|
-
WHERE source = 'CWE'
|
64
|
-
GROUP BY dvr.vulnerability_id, dvr.reference
|
65
|
-
) cwe USING (vulnerability_id)
|
66
|
-
LEFT OUTER JOIN
|
67
|
-
(SELECT DISTINCT ON (vulnerability_id)
|
68
|
-
vulnerability_id,
|
69
|
-
dvr.reference AS ref
|
70
|
-
FROM dim_vulnerability_reference dvr
|
71
|
-
WHERE source = 'CVE'
|
72
|
-
GROUP BY dvr.vulnerability_id, dvr.reference
|
73
|
-
) cve USING (vulnerability_id)
|
74
|
-
LEFT OUTER JOIN (SELECT DISTINCT ON (dvc.vulnerability_id)
|
75
|
-
dvc.vulnerability_id,
|
76
|
-
dvc.category_name AS category
|
77
|
-
FROM dim_vulnerability_category dvc
|
78
|
-
GROUP BY dvc.vulnerability_id, dvc.category_name) dvc USING (vulnerability_id)
|
79
|
-
LEFT OUTER JOIN (SELECT
|
80
|
-
dvr.vulnerability_id,
|
81
|
-
string_agg(dvr.source || ': ' || dvr.reference, '|') AS references
|
82
|
-
FROM dim_vulnerability_reference dvr
|
83
|
-
GROUP BY dvr.vulnerability_id) ref USING (vulnerability_id)
|
84
|
-
LEFT OUTER JOIN (SELECT
|
85
|
-
vulnerability_id,
|
86
|
-
array_agg(solution_id) AS solutions
|
87
|
-
FROM dim_vulnerability_solution
|
88
|
-
GROUP BY vulnerability_id) sol USING (vulnerability_id)
|
89
|
-
WHERE date_modified >= '#{options[:vuln_query_date]}'"
|
90
|
-
end
|
91
|
-
|
92
|
-
def self.vulnerability_references(options={})
|
93
|
-
"SELECT concat('R7_', vulnerability_id) as ID,
|
94
|
-
Source,
|
95
|
-
Reference
|
96
|
-
FROM (SELECT vulnerability_id
|
97
|
-
FROM dim_vulnerability
|
98
|
-
WHERE date_modified >= '#{options[:vuln_query_date]}') dv
|
99
|
-
JOIN (SELECT vulnerability_id, Source, Reference
|
100
|
-
FROM dim_vulnerability_reference) dvr USING (vulnerability_id)"
|
101
|
-
end
|
102
|
-
|
103
|
-
def self.vulnerability_category(options={})
|
104
|
-
"SELECT concat('R7_', vulnerability_id) as ID, dvc.Category
|
105
|
-
FROM dim_vulnerability
|
106
|
-
LEFT OUTER JOIN
|
107
|
-
(SELECT vulnerability_id, category_name as Category
|
108
|
-
FROM dim_vulnerability_category dvc) dvc USING (vulnerability_id)
|
109
|
-
WHERE date_modified >= '#{options[:vuln_query_date]}'"
|
110
|
-
end
|
111
|
-
|
112
|
-
def self.assets(options={})
|
113
|
-
last_import_date = options[:delta]
|
114
|
-
site_id = options[:site_id]
|
115
|
-
|
116
|
-
"WITH scan_found AS (
|
117
|
-
SELECT DISTINCT (asset_id)
|
118
|
-
asset_id,
|
119
|
-
MIN(scan_id) AS scan_found
|
120
|
-
FROM fact_asset_event
|
121
|
-
WHERE type = 'SCAN'
|
122
|
-
GROUP BY asset_id
|
123
|
-
), new_assets AS (
|
124
|
-
SELECT
|
125
|
-
sf.asset_id
|
126
|
-
FROM dim_scan ds
|
127
|
-
JOIN scan_found sf ON (ds.scan_id = sf.scan_found)
|
128
|
-
WHERE ds.finished > '#{last_import_date}'
|
129
|
-
AND ds.site_id = #{site_id}
|
130
|
-
)
|
131
|
-
SELECT
|
132
|
-
coalesce(host_name, CAST(dim_asset.asset_id AS TEXT)) AS Name,
|
133
|
-
dim_asset.ip_address,
|
134
|
-
dim_asset.mac_address,
|
135
|
-
concat('Rapid7 Nexpose') AS Discovery_Source,
|
136
|
-
CAST(CASE
|
137
|
-
WHEN dim_asset.host_type = 'Virtual Machine' OR dim_asset.host_type = 'Hypervisor'
|
138
|
-
THEN 1
|
139
|
-
ELSE 0
|
140
|
-
END AS BIT) AS Is_Virtual,
|
141
|
-
CONCAT('\"', dim_asset.os_description, '\"') AS Operating_System,
|
142
|
-
dim_asset.last_assessed_for_vulnerabilities AS Most_Recent_Discovery,
|
143
|
-
dim_asset.asset_id AS Nexpose_ID,
|
144
|
-
fact_asset.pci_status,
|
145
|
-
dim_asset.credential_status
|
146
|
-
FROM dim_asset
|
147
|
-
JOIN fact_asset USING (asset_id)
|
148
|
-
WHERE dim_asset.asset_id IN (
|
149
|
-
SELECT asset_id
|
150
|
-
FROM new_assets
|
151
|
-
)"
|
152
|
-
end
|
153
|
-
|
154
|
-
def self.generate_vulnerability_filter(cves, cvss_range, cvss_strings)
|
155
|
-
return '' if cves.to_a.empty? && cvss_range.to_a.empty?
|
156
|
-
vuln_filter = ''
|
157
|
-
vuln_filter += cves.map { |c| "reference='#{c}'" }.join(' OR ') unless cves.nil?
|
158
|
-
|
159
|
-
cvss_min = cvss_range.first || '0'
|
160
|
-
cvss_max = cvss_range.last || '10'
|
161
|
-
|
162
|
-
cvss_score = "(coalesce(cvss#{cvss_strings[:choice]}_score,
|
163
|
-
cvss#{cvss_strings[:fallback]}_score))"
|
164
|
-
|
165
|
-
unless cvss_min.to_s == '0' && cvss_max.to_s == '10'
|
166
|
-
vuln_filter += ' AND ' unless vuln_filter.empty?
|
167
|
-
vuln_filter += "#{cvss_score} >= #{cvss_min} AND #{cvss_score} <= #{cvss_max}"
|
168
|
-
end
|
169
|
-
return '' if vuln_filter.empty?
|
170
|
-
|
171
|
-
"AND favi.vulnerability_id IN (
|
172
|
-
SELECT dvr.vulnerability_id
|
173
|
-
FROM dim_vulnerability_reference dvr
|
174
|
-
JOIN dim_vulnerability dv ON dv.vulnerability_id = dvr.vulnerability_id
|
175
|
-
WHERE dvr.vulnerability_id IN (SELECT av.vulnerability_id FROM asset_vulns av)
|
176
|
-
AND #{vuln_filter})"
|
177
|
-
end
|
178
|
-
|
179
|
-
def self.generate_date_filter(dates)
|
180
|
-
return '' if dates.to_a.empty?
|
181
|
-
filters = []
|
182
|
-
filters << "first_found > '#{dates.first}'" unless dates.first.nil?
|
183
|
-
filters << "first_found < '#{dates.last}'" unless dates.last.nil?
|
184
|
-
filters.empty? ? '' : " AND #{filters.join(' AND ')}"
|
185
|
-
end
|
186
|
-
|
187
|
-
def self.vulnerable_new_items(options={})
|
188
|
-
site_id = options[:site_id]
|
189
|
-
last_import_date = options[:delta]
|
190
|
-
vuln_filter = self.generate_vulnerability_filter(
|
191
|
-
options[:filters][:cve], options[:filters][:cvss], options[:cvss_v]
|
192
|
-
)
|
193
|
-
date_filters = self.generate_date_filter(options[:filters][:date])
|
194
|
-
|
195
|
-
"WITH asset_vulns AS (
|
196
|
-
SELECT DISTINCT ON (favf.asset_id, favf.vulnerability_id)
|
197
|
-
favf.asset_id,
|
198
|
-
favf.vulnerability_id,
|
199
|
-
favf.date AS first_found,
|
200
|
-
favf.vulnerability_instances,
|
201
|
-
ip_address,
|
202
|
-
(SELECT max(date)
|
203
|
-
FROM fact_asset_event fae
|
204
|
-
WHERE type = 'SCAN'
|
205
|
-
AND fae.asset_id = favf.asset_id
|
206
|
-
GROUP BY fae.asset_id) AS last_found
|
207
|
-
FROM fact_asset_vulnerability_finding favf
|
208
|
-
JOIN dim_site_asset dsa ON favf.asset_id = dsa.asset_id AND dsa.site_id = #{site_id}
|
209
|
-
JOIN dim_asset da ON favf.asset_id = da.asset_id
|
210
|
-
WHERE date > '#{last_import_date}'
|
211
|
-
), valid_vulns AS (
|
212
|
-
SELECT DISTINCT ON (favi.asset_id, favi.vulnerability_id)
|
213
|
-
favi.asset_id,
|
214
|
-
favi.vulnerability_id,
|
215
|
-
array_agg(DISTINCT favi.port) AS ports,
|
216
|
-
array_agg(DISTINCT davs.solution_id) AS solutions,
|
217
|
-
favi.protocol,
|
218
|
-
regexp_replace(htmltotext(favi.proof), '\"','''', 'g') AS proof,
|
219
|
-
favi.status
|
220
|
-
FROM fact_asset_vulnerability_instance favi
|
221
|
-
LEFT JOIN dim_asset_vulnerability_finding_solution davs ON
|
222
|
-
davs.asset_id = favi.asset_id AND davs.vulnerability_id = favi.vulnerability_id
|
223
|
-
WHERE (favi.vulnerability_id, favi.asset_id) IN (SELECT av.vulnerability_id, av.asset_id FROM asset_vulns av)
|
224
|
-
#{vuln_filter}
|
225
|
-
GROUP BY favi.asset_id, favi.vulnerability_id, protocol, proof, status
|
226
|
-
)
|
227
|
-
SELECT
|
228
|
-
CAST(nv.asset_id AS TEXT) AS Configuration_Item,
|
229
|
-
'true' AS Active,
|
230
|
-
concat('R7_', nv.vulnerability_id) AS Vulnerability,
|
231
|
-
nv.first_found AS First_Found,
|
232
|
-
nv.last_found AS Last_Found,
|
233
|
-
nv.vulnerability_instances AS Times_Found,
|
234
|
-
nv.ip_address AS IP_Address,
|
235
|
-
coalesce(NULLIF(CONCAT('\"', vv.proof, '\"'), '\"\"'), 'None') AS Proof,
|
236
|
-
vv.status AS Status,
|
237
|
-
array_to_string(vv.ports, ' ', '') AS Ports,
|
238
|
-
coalesce(vv.protocol, 'None') AS Protocol,
|
239
|
-
NULLIF(CONCAT('\"', array_to_string(vv.solutions, ',', ''), '\"'), '\"\"') AS Solutions
|
240
|
-
FROM asset_vulns nv
|
241
|
-
JOIN valid_vulns vv ON vv.asset_id = nv.asset_id AND vv.vulnerability_id = nv.vulnerability_id
|
242
|
-
#{date_filters}
|
243
|
-
ORDER BY nv.asset_id, Vulnerability"
|
244
|
-
end
|
245
|
-
|
246
|
-
def self.vulnerable_old_items(options={})
|
247
|
-
site_id = options[:site_id]
|
248
|
-
last_import_date = options[:delta]
|
249
|
-
vuln_filter = self.generate_vulnerability_filter(
|
250
|
-
options[:filters][:cve], options[:filters][:cvss], options[:cvss_v]
|
251
|
-
)
|
252
|
-
date_filters = self.generate_date_filter(options[:filters][:date])
|
253
|
-
|
254
|
-
"WITH site_assets AS (
|
255
|
-
SELECT asset_id
|
256
|
-
FROM dim_site_asset
|
257
|
-
WHERE site_id = #{site_id}
|
258
|
-
), asset_vulns AS (
|
259
|
-
-- Asset vulns at last import
|
260
|
-
SELECT asset_id, vulnerability_id, date AS first_found
|
261
|
-
FROM fact_asset_vulnerability_finding_date
|
262
|
-
WHERE asset_id IN (SELECT asset_id FROM site_assets)
|
263
|
-
AND DAY = (SELECT periodbefore('#{last_import_date}'))
|
264
|
-
), current_vulns AS (
|
265
|
-
SELECT asset_id, vulnerability_id
|
266
|
-
FROM fact_asset_vulnerability_finding
|
267
|
-
WHERE asset_id IN (SELECT asset_id FROM site_assets)
|
268
|
-
)
|
269
|
-
SELECT
|
270
|
-
CAST(da.asset_id AS TEXT) AS Configuration_Item,
|
271
|
-
'false' AS Active,
|
272
|
-
concat('R7_', favi.vulnerability_id) AS Vulnerability,
|
273
|
-
da.ip_address AS IP_Address
|
274
|
-
FROM dim_asset da
|
275
|
-
-- Name this table favi to use the vulnerability ID with the vuln filter.
|
276
|
-
JOIN asset_vulns favi ON (da.asset_id = favi.asset_id)
|
277
|
-
LEFT JOIN current_vulns cv ON (cv.asset_id = da.asset_id) AND (cv.vulnerability_id = favi.vulnerability_id)
|
278
|
-
WHERE da.asset_id IN (SELECT asset_id FROM site_assets)
|
279
|
-
AND cv.vulnerability_id IS NULL
|
280
|
-
#{vuln_filter}
|
281
|
-
#{date_filters}
|
282
|
-
ORDER BY da.asset_id, Vulnerability"
|
283
|
-
end
|
284
|
-
|
285
|
-
def self.vulnerability_solutions(options={})
|
286
|
-
"SELECT DISTINCT (solution_id)
|
287
|
-
solution_id,
|
288
|
-
nexpose_id,
|
289
|
-
coalesce(NULLIF(CONCAT('\"',regexp_replace(htmltotext(fix), '\"','''', 'g'),'\"'), '\"\"'), 'None') as fix,
|
290
|
-
estimate,
|
291
|
-
coalesce(NULLIF(CONCAT('\"',regexp_replace(htmltotext(summary), '\"','''', 'g'),'\"'), '\"\"'), 'None') as summary,
|
292
|
-
solution_type,
|
293
|
-
coalesce(NULLIF(CONCAT('\"',regexp_replace(htmltotext(applies_to), '\"','''', 'g'),'\"'), '\"\"'), 'None') as applies_to,
|
294
|
-
coalesce(NULLIF(CONCAT('\"', url, '\"'), '\"\"'), 'None') as url,
|
295
|
-
coalesce(NULLIF(CONCAT('\"',regexp_replace(htmltotext(additional_data), '\"','''', 'g'),'\"'), '\"\"'), 'None') as additional_data,
|
296
|
-
CONCAT('\"', array_to_string(req_solutions, ',', ''), '\"') as required_solutions,
|
297
|
-
CONCAT('\"', array_to_string(super_solutions, ',', ''), '\"') as superceding_solutions
|
298
|
-
FROM dim_solution
|
299
|
-
RIGHT OUTER JOIN (
|
300
|
-
SELECT DISTINCT (solution_id) solution_id
|
301
|
-
FROM (
|
302
|
-
SELECT solution_id, vulnerability_id, date_modified
|
303
|
-
FROM dim_vulnerability
|
304
|
-
LEFT JOIN dim_vulnerability_solution idvs USING (vulnerability_id)
|
305
|
-
) dvs
|
306
|
-
WHERE date_modified >= '#{options[:vuln_query_date]}'
|
307
|
-
UNION
|
308
|
-
SELECT DISTINCT (solution_id) solution_id
|
309
|
-
FROM dim_solution
|
310
|
-
LEFT JOIN (
|
311
|
-
SELECT solution_id, vulnerability_id, date_modified
|
312
|
-
FROM dim_vulnerability
|
313
|
-
LEFT JOIN dim_vulnerability_solution idvs USING (vulnerability_id)
|
314
|
-
) ndvs USING (solution_id)
|
315
|
-
WHERE vulnerability_id IS NULL
|
316
|
-
) dvs USING (solution_id)
|
317
|
-
LEFT JOIN (
|
318
|
-
SELECT DISTINCT (solution_id) solution_id,
|
319
|
-
array_agg(required_solution_id) as req_solutions
|
320
|
-
FROM dim_solution_prerequisite
|
321
|
-
GROUP BY solution_id
|
322
|
-
) dsp USING (solution_id)
|
323
|
-
JOIN (
|
324
|
-
SELECT DISTINCT (solution_id) solution_id,
|
325
|
-
array_agg(superceding_solution_id) as super_solutions
|
326
|
-
FROM dim_solution_highest_supercedence
|
327
|
-
GROUP BY solution_id
|
328
|
-
) dshs USING (solution_id)
|
329
|
-
ORDER BY solution_id"
|
330
|
-
end
|
331
|
-
|
332
|
-
def self.method_missing(m, *args, &block)
|
333
|
-
msg = "Method #{m} is not yet available for data warehouse mode."
|
334
|
-
|
335
|
-
log = NexposeServiceNow::NxLogger.instance
|
336
|
-
log.log_message(msg)
|
337
|
-
puts "#{msg} Exiting..."
|
338
|
-
exit 0
|
339
|
-
end
|
340
|
-
end
|
341
|
-
end
|