nexpose_ticketing 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -13
- data/bin/nexpose_servicenow +1 -1
- data/lib/nexpose_ticketing/config/servicenow.config +2 -2
- data/lib/nexpose_ticketing/config/ticket_service.config +6 -2
- data/lib/nexpose_ticketing/helpers/servicenow_helper.rb +12 -7
- data/lib/nexpose_ticketing/queries.rb +5 -5
- data/lib/nexpose_ticketing/report_helper.rb +76 -0
- data/lib/nexpose_ticketing/ticket_repository.rb +76 -21
- data/lib/nexpose_ticketing/ticket_service.rb +123 -41
- metadata +10 -10
- data/nexpose_ticketing.gemspec +0 -20
checksums.yaml
CHANGED
@@ -1,15 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
5
|
-
data.tar.gz: !binary |-
|
6
|
-
YTczODY0MjMzYTg3OTQwMTA1ZDZmMjk1ODdhZmJiZGQ2MGUxOTRjNA==
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 8b5571a811b41e5db9bad42f938687414f521da3
|
4
|
+
data.tar.gz: 97c1ea6f3932f5b37138f15128f2530eb6bae9f1
|
7
5
|
SHA512:
|
8
|
-
metadata.gz:
|
9
|
-
|
10
|
-
MDRlOWZhYTgyZGY2ZWJmZjhlYmYyYWMzMGE4ZmVjOTZlNjNiYmY5NTEwNmI4
|
11
|
-
NGQ5OWZhMzExYTAyYjg0Y2M5ZjViNmYwYzg2ZGI2NGIxZGRjNzc=
|
12
|
-
data.tar.gz: !binary |-
|
13
|
-
ZmFiNWE4MjAzMzk1N2ExMTFiMWNmZTM4NjIxNjk1MzE2ZjE2OTVmOGVlNjU3
|
14
|
-
ZGM5MmY0YzZkMmZkZjViMWIwMjFhZTJhM2M3NWIzZmUwNTNiNzk0MjJiMWQy
|
15
|
-
OWVkZjY2ZmYzNTkxMGJlOWNkMGU4ZjUxNjkwYTYyYTI2N2M0N2Y=
|
6
|
+
metadata.gz: a34d4c2ae876184066c674ba39dfa7c2fa792b476c10722d2da0dcd2e4ef8516062f5dfcd8e317a80fc4abfd8001b4b3b8f32cd61d6cf10a1012e561919dd217
|
7
|
+
data.tar.gz: a039db4db30f7fdf24a9f56c3851533358085226da163f8a8857abbabce71637e38029c39f48e334926a931abd3c4a0c099058b7e173edc67e94c01d0e181c5c
|
data/bin/nexpose_servicenow
CHANGED
@@ -7,11 +7,11 @@
|
|
7
7
|
:helper_name: ServiceNowHelper
|
8
8
|
|
9
9
|
# (M) ServiceNow url (currently requires using JSON)
|
10
|
-
:servicenow_url: https://
|
10
|
+
:servicenow_url: https://your.service-now.com/incident.do?JSON
|
11
11
|
# (M) Username for ServiceNow
|
12
12
|
:username: admin
|
13
13
|
# (M) Password for above username
|
14
|
-
:password:
|
14
|
+
:password: admin
|
15
15
|
# (M) If 'Y', SSL connections will output to stderr (default is 'N')
|
16
16
|
:verbose_mode: N
|
17
17
|
# (M) Amount of times the helper will follow 301/302 redirections
|
@@ -8,7 +8,7 @@
|
|
8
8
|
:logging_enabled: true
|
9
9
|
# Filters the reports to specific sites one per line, leave empty for no site.
|
10
10
|
:sites:
|
11
|
-
- '
|
11
|
+
- '7'
|
12
12
|
# Minimum floor severity to report on. Number between 0 and 10.
|
13
13
|
:severity: 8
|
14
14
|
# (M) Name of the report historial file saved in disk.
|
@@ -17,10 +17,14 @@
|
|
17
17
|
# 'D' Default IP *-* Vulnerability
|
18
18
|
# 'I' IP address -* Vulnerability
|
19
19
|
:ticket_mode: I
|
20
|
+
# Timeout in seconds. The number of seconds the GEM waits for a response from Nexpose before exiting.
|
21
|
+
:timeout: 10800
|
22
|
+
# Ticket batching. Breaks ticket processing into groups of value size controlling resource utilisation of both systems.
|
23
|
+
:batch_size: 300
|
20
24
|
# Nexpose options.
|
21
25
|
:nexpose_data:
|
22
26
|
# (M) Nexpose console hostname.
|
23
|
-
:nxconsole:
|
27
|
+
:nxconsole: nexpose-rh.dev.lax.rapid7.com
|
24
28
|
# (M) Nexpose username.
|
25
29
|
:nxuser: nxadmin
|
26
30
|
# (M) Nexpose password.
|
@@ -176,7 +176,7 @@ class ServiceNowHelper
|
|
176
176
|
tickets = []
|
177
177
|
current_ip = -1
|
178
178
|
CSV.parse(vulnerability_list.chomp, headers: :first_row) do |row|
|
179
|
-
if current_ip == -1
|
179
|
+
if current_ip == -1
|
180
180
|
current_ip = row['ip_address']
|
181
181
|
@log.log_message("Creating ticket with IP address: #{row['ip_address']}")
|
182
182
|
@ticket = {
|
@@ -187,24 +187,25 @@ class ServiceNowHelper
|
|
187
187
|
'urgency' => '1',
|
188
188
|
'short_description' => "#{row['ip_address']} => Vulnerabilities",
|
189
189
|
'work_notes' => "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
190
|
-
++ New Vulnerabilities
|
190
|
+
++ New Vulnerabilities ++++++++++++++++++++++++++++++++++++
|
191
191
|
+++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
|
192
192
|
}
|
193
193
|
end
|
194
194
|
if current_ip == row['ip_address']
|
195
195
|
@ticket['work_notes'] +=
|
196
|
-
"==========================================
|
196
|
+
"\n\n==========================================
|
197
197
|
Summary: #{row['summary']}
|
198
198
|
----------------------------------------------------------------------------
|
199
199
|
Fix: #{row['fix']}"
|
200
200
|
unless row['url'].nil?
|
201
201
|
@ticket['work_notes'] +=
|
202
|
-
"----------------------------------------------------------------------------
|
202
|
+
"\n----------------------------------------------------------------------------
|
203
203
|
URL: #{row['url']}"
|
204
204
|
end
|
205
205
|
end
|
206
206
|
unless current_ip == row['ip_address']
|
207
207
|
# NXID in the work_notes is the unique identifier used to query incidents to update them.
|
208
|
+
@log.log_message("Found new IP address. Finishing ticket with with IP address: #{current_ip} and moving onto IP #{row['ip_address']}")
|
208
209
|
@ticket['work_notes'] += "\nNXID: #{current_ip}"
|
209
210
|
@ticket = @ticket.to_json
|
210
211
|
tickets.push(@ticket)
|
@@ -213,7 +214,7 @@ class ServiceNowHelper
|
|
213
214
|
end
|
214
215
|
end
|
215
216
|
# NXID in the work_notes is the unique identifier used to query incidents to update them.
|
216
|
-
@ticket['work_notes'] += "\nNXID: #{current_ip}"
|
217
|
+
@ticket['work_notes'] += "\nNXID: #{current_ip}" unless (@ticket.size == 0)
|
217
218
|
tickets.push(@ticket.to_json) unless @ticket.nil?
|
218
219
|
tickets
|
219
220
|
end
|
@@ -242,8 +243,12 @@ class ServiceNowHelper
|
|
242
243
|
ticket_status = row['comparison']
|
243
244
|
@log.log_message("Creating ticket update with IP address: #{row['ip_address']}")
|
244
245
|
@log.log_message("Ticket status #{ticket_status}")
|
246
|
+
action = 'update'
|
247
|
+
if ticket_status == 'New'
|
248
|
+
action = 'insert'
|
249
|
+
end
|
245
250
|
@ticket = {
|
246
|
-
'sysparm_action' =>
|
251
|
+
'sysparm_action' => action,
|
247
252
|
'sysparm_query' => "work_notesCONTAINSNXID: #{row['ip_address']}",
|
248
253
|
'work_notes' =>
|
249
254
|
"\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
@@ -283,7 +288,7 @@ class ServiceNowHelper
|
|
283
288
|
end
|
284
289
|
end
|
285
290
|
# NXID in the work_notes is the unique identifier used to query incidents to update them.
|
286
|
-
@ticket['work_notes'] += "\nNXID: #{current_ip}"
|
291
|
+
@ticket['work_notes'] += "\nNXID: #{current_ip}" unless (@ticket.size == 0)
|
287
292
|
tickets.push(@ticket.to_json) unless @ticket.nil?
|
288
293
|
tickets
|
289
294
|
end
|
@@ -19,7 +19,7 @@ module NexposeTicketing
|
|
19
19
|
# |url| |summary| |fix|
|
20
20
|
#
|
21
21
|
def self.all_new_vulns
|
22
|
-
|
22
|
+
"SELECT DISTINCT on (da.ip_address, davs.solution_id) subs.asset_id, da.ip_address, subs.current_scan, subs.vulnerability_id, davs.solution_id, ds.nexpose_id,
|
23
23
|
ds.url,proofAsText(ds.summary) as summary, proofAsText(ds.fix) as fix
|
24
24
|
FROM (SELECT fasv.asset_id, fasv.vulnerability_id, s.current_scan
|
25
25
|
FROM fact_asset_scan_vulnerability_finding fasv
|
@@ -28,14 +28,14 @@ module NexposeTicketing
|
|
28
28
|
SELECT asset_id, previousScan(asset_id) AS baseline_scan, lastScan(asset_id) AS current_scan
|
29
29
|
FROM dim_asset) s
|
30
30
|
ON s.asset_id = fasv.asset_id AND (fasv.scan_id = s.baseline_scan OR fasv.scan_id = s.current_scan)
|
31
|
-
GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan
|
32
|
-
HAVING baselineComparison(fasv.scan_id, current_scan) = '
|
31
|
+
GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan, fasv.scan_id
|
32
|
+
HAVING NOT baselineComparison(fasv.scan_id, current_scan) = 'Old'
|
33
33
|
) subs
|
34
34
|
JOIN dim_asset_vulnerability_solution davs USING (vulnerability_id)
|
35
35
|
JOIN dim_solution ds USING (solution_id)
|
36
36
|
JOIN dim_asset da ON subs.asset_id = da.asset_id
|
37
|
-
ORDER BY da.ip_address"
|
38
|
-
|
37
|
+
ORDER BY da.ip_address, davs.solution_id"
|
38
|
+
end
|
39
39
|
|
40
40
|
# Gets all new vulnerabilities happening after a reported scan id.
|
41
41
|
#
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require 'nexpose'
|
2
|
+
require 'securerandom'
|
3
|
+
include Nexpose
|
4
|
+
|
5
|
+
module NexposeReportHelper
|
6
|
+
class ReportOps
|
7
|
+
|
8
|
+
def initialize(nsc, timeout)
|
9
|
+
@timeout = timeout
|
10
|
+
@nsc = nsc
|
11
|
+
end
|
12
|
+
|
13
|
+
def generate_sql_report_config
|
14
|
+
random_name = "Nexpose-ticketing-Temp-#{SecureRandom.uuid}"
|
15
|
+
Nexpose::ReportConfig.new(random_name, nil, 'sql')
|
16
|
+
end
|
17
|
+
|
18
|
+
def save_generate_cleanup_report_config(report_config)
|
19
|
+
report_id = report_config.save(@nsc, false)
|
20
|
+
@nsc.generate_report(report_id, true)
|
21
|
+
wait_for_report(report_id)
|
22
|
+
report_details = @nsc.last_report(report_id)
|
23
|
+
file = Tempfile.new("#{report_id}")
|
24
|
+
file.write(@nsc.download(report_details.uri))
|
25
|
+
#Got the report, cleanup server-side
|
26
|
+
@nsc.delete_report_config(report_id)
|
27
|
+
file
|
28
|
+
end
|
29
|
+
|
30
|
+
# Wait for report generation to complete.
|
31
|
+
#
|
32
|
+
# @param [Fixnum] id Report configuration ID of the report waiting to generate.
|
33
|
+
#
|
34
|
+
def wait_for_report(id)
|
35
|
+
wait_until(:fail_on_exceptions => TRUE, :on_timeout => "Report generation timed out. Status: #{r = @nsc.last_report(id); r ? r.status : 'unknown'}") {
|
36
|
+
if %w(Failed Aborted Unknown).include?(@nsc.last_report(id).status)
|
37
|
+
raise "Report failed to generate! Status <#{@nsc.last_report(id).status}>"
|
38
|
+
end
|
39
|
+
@nsc.last_report(id).status == 'Generated'
|
40
|
+
}
|
41
|
+
end
|
42
|
+
|
43
|
+
# Wait for a given block to evaluate to true.
|
44
|
+
#
|
45
|
+
# The following flags are accepted as arguments:
|
46
|
+
# :timeout Number of seconds to wait before timing out. Defaults to 90.
|
47
|
+
# :polling_interval Number of seconds to wait between checking block again.
|
48
|
+
# Defaults to 5.
|
49
|
+
# :fail_on_exceptions Whether to fail fast. Defaults to off.
|
50
|
+
# :on_timeout Message to raise with the exception when a timeout occurs.
|
51
|
+
#
|
52
|
+
# Example usage:
|
53
|
+
# wait_until { site.id > 0 }
|
54
|
+
# wait_until(:timeout => 30, :polling_interval => 0.5) { 1 == 2 }
|
55
|
+
# wait_until(:on_timeout => 'Unable to confirm scan integration.') { console.sites.find { |site| site[:site_id] == site_id.to_i }[:risk_score] > 0.0 }
|
56
|
+
#
|
57
|
+
def wait_until(options = {})
|
58
|
+
polling_interval = 90
|
59
|
+
time_limit = Time.now + @timeout
|
60
|
+
loop do
|
61
|
+
begin
|
62
|
+
val = yield
|
63
|
+
return val if val
|
64
|
+
rescue Exception => error
|
65
|
+
raise error if options[:fail_on_exceptions]
|
66
|
+
end
|
67
|
+
if Time.now >= time_limit
|
68
|
+
raise options[:on_timeout] if options[:on_timeout]
|
69
|
+
error ||= 'Timed out waiting for condition.'
|
70
|
+
raise error
|
71
|
+
end
|
72
|
+
sleep polling_interval
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -4,10 +4,28 @@ module NexposeTicketing
|
|
4
4
|
require 'csv'
|
5
5
|
require 'nexpose'
|
6
6
|
require 'nexpose_ticketing/queries'
|
7
|
+
require 'nexpose_ticketing/report_helper'
|
8
|
+
|
9
|
+
@timeout = 10800
|
10
|
+
|
11
|
+
def initialize(options = nil)
|
12
|
+
@timeout = options[:timeout]
|
13
|
+
end
|
7
14
|
|
8
15
|
def nexpose_login(nexpose_data)
|
9
16
|
@nsc = Nexpose::Connection.new(nexpose_data[:nxconsole], nexpose_data[:nxuser], nexpose_data[:nxpasswd])
|
10
17
|
@nsc.login
|
18
|
+
#After login, create the report helper
|
19
|
+
@report_helper = NexposeReportHelper::ReportOps.new(@nsc, @timeout)
|
20
|
+
end
|
21
|
+
|
22
|
+
# Returns an array of all sites in the users environment.
|
23
|
+
#
|
24
|
+
# * *Returns* :
|
25
|
+
# - An array of Nexpose::SiteSummary objects.
|
26
|
+
#
|
27
|
+
def all_site_details()
|
28
|
+
@nsc.sites
|
11
29
|
end
|
12
30
|
|
13
31
|
# Reads the site scan history from disk.
|
@@ -31,28 +49,56 @@ module NexposeTicketing
|
|
31
49
|
# * *Args* :
|
32
50
|
# - +csv_file_name+ - CSV File name.
|
33
51
|
#
|
34
|
-
def save_last_scans(csv_file_name,
|
52
|
+
def save_last_scans(csv_file_name, options = {}, saved_file = nil)
|
53
|
+
current_scan_state = load_last_scans(options)
|
54
|
+
save_scans_to_file(csv_file_name, current_scan_state, saved_file)
|
55
|
+
end
|
56
|
+
|
57
|
+
# Loads the last scan info to memory.
|
58
|
+
#
|
59
|
+
# * *Args* :
|
60
|
+
# - +csv_file_name+ - CSV File name.
|
61
|
+
#
|
62
|
+
def load_last_scans(options = {}, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
|
63
|
+
sites = Array(options[:sites])
|
35
64
|
report_config.add_filter('version', '1.2.0')
|
36
65
|
report_config.add_filter('query', Queries.last_scans)
|
37
|
-
|
66
|
+
|
67
|
+
report_output = report_config.generate(@nsc, @timeout)
|
38
68
|
csv_output = CSV.parse(report_output.chomp, headers: :first_row)
|
39
|
-
|
69
|
+
|
70
|
+
#We only care about sites we are monitoring.
|
71
|
+
trimmed_csv = []
|
72
|
+
trimmed_csv << report_output.lines.first
|
73
|
+
csv_output.each do |row|
|
74
|
+
if sites.include? row.to_s[0]
|
75
|
+
trimmed_csv << row
|
76
|
+
end
|
77
|
+
end
|
78
|
+
trimmed_csv
|
79
|
+
end
|
80
|
+
|
81
|
+
# Saves CSV scan information to disk
|
82
|
+
#
|
83
|
+
# * *Args* :
|
84
|
+
# - +csv_file_name+ - CSV File name.
|
85
|
+
#
|
86
|
+
def save_scans_to_file(csv_file_name, trimmed_csv, saved_file = nil)
|
87
|
+
saved_file.open(csv_file_name, 'w') { |file| file.puts(trimmed_csv) } unless saved_file.nil?
|
40
88
|
if saved_file.nil?
|
41
|
-
File.open(csv_file_name, 'w') { |file| file.puts(
|
89
|
+
File.open(csv_file_name, 'w') { |file| file.puts(trimmed_csv) }
|
42
90
|
end
|
43
91
|
end
|
44
92
|
|
45
|
-
# Gets the last scan information from nexpose.
|
93
|
+
# Gets the last scan information from nexpose sans the CSV headers.
|
46
94
|
#
|
47
95
|
# * *Returns* :
|
48
96
|
# - A hash with site_ids => last_scan_id
|
49
97
|
#
|
50
|
-
def last_scans(
|
51
|
-
report_config.add_filter('version', '1.2.0')
|
52
|
-
report_config.add_filter('query', Queries.last_scans)
|
53
|
-
report_output = report_config.generate(@nsc).chomp
|
98
|
+
def last_scans(options = {})
|
54
99
|
nexpose_sites = Hash.new(-1)
|
55
|
-
|
100
|
+
trimmed_csv = load_last_scans(options)
|
101
|
+
trimmed_csv.drop(1).each do |row|
|
56
102
|
nexpose_sites[row['site_id']] = row['last_scan_id'].to_i
|
57
103
|
end
|
58
104
|
nexpose_sites
|
@@ -62,23 +108,29 @@ module NexposeTicketing
|
|
62
108
|
#
|
63
109
|
# * *Args* :
|
64
110
|
# - +site_options+ - A Hash with site(s) and severity level.
|
111
|
+
# - +site_to_query+ - Override for user-configured site options
|
65
112
|
#
|
66
113
|
# * *Returns* :
|
67
114
|
# - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
|
68
115
|
# |url| |summary| |fix|
|
69
116
|
#
|
70
|
-
def all_vulns(
|
71
|
-
|
72
|
-
|
117
|
+
def all_vulns(options = {}, site_override = nil)
|
118
|
+
if site_override.nil?
|
119
|
+
sites = Array(options[:sites])
|
120
|
+
else
|
121
|
+
sites = site_override
|
122
|
+
end
|
123
|
+
report_config = @report_helper.generate_sql_report_config()
|
124
|
+
severity = options[:severity].nil? ? 0 : options[:severity]
|
73
125
|
report_config.add_filter('version', '1.2.0')
|
74
126
|
report_config.add_filter('query', Queries.all_new_vulns)
|
75
127
|
unless sites.empty?
|
76
|
-
sites.each do |site_id|
|
128
|
+
Array(sites).each do |site_id|
|
77
129
|
report_config.add_filter('site', site_id)
|
78
130
|
end
|
79
131
|
end
|
80
132
|
report_config.add_filter('vuln-severity', severity)
|
81
|
-
|
133
|
+
@report_helper.save_generate_cleanup_report_config(report_config)
|
82
134
|
end
|
83
135
|
|
84
136
|
# Gets the new vulns from base scan reported_scan_id and the newest / latest scan from a site.
|
@@ -90,7 +142,8 @@ module NexposeTicketing
|
|
90
142
|
# - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
|
91
143
|
# |url| |summary| |fix|
|
92
144
|
#
|
93
|
-
def new_vulns_sites(site_options = {}
|
145
|
+
def new_vulns_sites(site_options = {})
|
146
|
+
report_config = @report_helper.generate_sql_report_config()
|
94
147
|
site = site_options[:site_id]
|
95
148
|
reported_scan_id = site_options[:scan_id]
|
96
149
|
fail 'Site cannot be null or empty' if site.nil? || reported_scan_id.nil?
|
@@ -99,7 +152,7 @@ module NexposeTicketing
|
|
99
152
|
report_config.add_filter('query', Queries.new_vulns_since_scan(reported_scan_id))
|
100
153
|
report_config.add_filter('site', site)
|
101
154
|
report_config.add_filter('vuln-severity', severity)
|
102
|
-
|
155
|
+
@report_helper.save_generate_cleanup_report_config(report_config)
|
103
156
|
end
|
104
157
|
|
105
158
|
# Gets the old vulns from base scan reported_scan_id and the newest / latest scan from a site.
|
@@ -111,7 +164,8 @@ module NexposeTicketing
|
|
111
164
|
# - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
|
112
165
|
# |url| |summary| |fix|
|
113
166
|
#
|
114
|
-
def old_vulns_sites(site_options = {}
|
167
|
+
def old_vulns_sites(site_options = {})
|
168
|
+
report_config = @report_helper.generate_sql_report_config()
|
115
169
|
site = site_options[:site_id]
|
116
170
|
reported_scan_id = site_options[:scan_id]
|
117
171
|
fail 'Site cannot be null or empty' if site.nil? || reported_scan_id.nil?
|
@@ -120,7 +174,7 @@ module NexposeTicketing
|
|
120
174
|
report_config.add_filter('query', Queries.old_vulns_since_scan(reported_scan_id))
|
121
175
|
report_config.add_filter('site', site)
|
122
176
|
report_config.add_filter('vuln-severity', severity)
|
123
|
-
|
177
|
+
@report_helper.save_generate_cleanup_report_config(report_config)
|
124
178
|
end
|
125
179
|
|
126
180
|
# Gets all vulns from base scan reported_scan_id and the newest / latest scan from a site. This is
|
@@ -133,7 +187,8 @@ module NexposeTicketing
|
|
133
187
|
# - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
|
134
188
|
# |url| |summary| |fix| |comparison|
|
135
189
|
#
|
136
|
-
def all_vulns_sites(site_options = {}
|
190
|
+
def all_vulns_sites(site_options = {})
|
191
|
+
report_config = @report_helper.generate_sql_report_config()
|
137
192
|
site = site_options[:site_id]
|
138
193
|
reported_scan_id = site_options[:scan_id]
|
139
194
|
fail 'Site cannot be null or empty' if site.nil? || reported_scan_id.nil?
|
@@ -142,7 +197,7 @@ module NexposeTicketing
|
|
142
197
|
report_config.add_filter('query', Queries.all_vulns_since_scan(reported_scan_id))
|
143
198
|
report_config.add_filter('site', site)
|
144
199
|
report_config.add_filter('vuln-severity', severity)
|
145
|
-
|
200
|
+
@report_helper.save_generate_cleanup_report_config(report_config)
|
146
201
|
end
|
147
202
|
end
|
148
203
|
end
|
@@ -81,9 +81,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
81
81
|
end
|
82
82
|
log_message("Enabling helper: #{@helper_data[:helper_name]}.")
|
83
83
|
@helper = eval(@helper_data[:helper_name]).new(@helper_data, @options)
|
84
|
-
|
84
|
+
|
85
|
+
log_message("Creating ticketing repository with timeout value: #{@options[:timeout]}.")
|
86
|
+
@ticket_repository = NexposeTicketing::TicketRepository.new(options)
|
85
87
|
@ticket_repository.nexpose_login(@nexpose_data)
|
86
|
-
@first_time = false
|
87
88
|
end
|
88
89
|
|
89
90
|
def setup_logging(enabled = false)
|
@@ -109,36 +110,40 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
109
110
|
log_message("Reading historical CSV file: #{historical_scan_file}.")
|
110
111
|
file_site_histories = ticket_repository.read_last_scans(historical_scan_file)
|
111
112
|
else
|
112
|
-
|
113
|
-
ticket_repository.save_last_scans(historical_scan_file)
|
114
|
-
log_message('Historical CSV file generated.')
|
115
|
-
file_site_histories = ticket_repository.read_last_scans(historical_scan_file)
|
116
|
-
@first_time = true
|
113
|
+
file_site_histories = nil
|
117
114
|
end
|
118
115
|
file_site_histories
|
119
116
|
end
|
120
117
|
|
121
118
|
# Generates a full site(s) report ticket(s).
|
122
|
-
def all_site_report(ticket_repository, options, helper
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
119
|
+
def all_site_report(ticket_repository, options, helper)
|
120
|
+
|
121
|
+
sites_to_query = Array.new
|
122
|
+
if options[:sites].empty?
|
123
|
+
log_message('No site(s) specified, generating full vulnerability report.')
|
124
|
+
@ticket_repository.all_site_details.each { |site| sites_to_query << site.id }
|
125
|
+
else
|
126
|
+
log_message('Generating full vulnerability report on user entered sites.')
|
127
|
+
sites_to_query = Array(options[:sites])
|
128
|
+
end
|
129
|
+
|
130
|
+
log_message("Generating full vulnerability report on the following sites: #{sites_to_query.join(', ')}")
|
131
|
+
|
132
|
+
sites_to_query.each { |site|
|
133
|
+
log_message("Running full vulnerability report on site #{site}")
|
134
|
+
all_vulns_file = ticket_repository.all_vulns(options, site)
|
135
|
+
log_message('Preparing tickets.')
|
136
|
+
ticket_rate_limiter(options, all_vulns_file, Proc.new { |ticket_batch| helper.prepare_create_tickets(ticket_batch) }, Proc.new { |tickets| helper.create_tickets(tickets) })
|
137
|
+
}
|
138
|
+
log_message('Finished process all vulnerabilities.')
|
133
139
|
end
|
134
140
|
|
135
141
|
# There's possibly a new scan with new data.
|
136
|
-
def delta_site_report(ticket_repository, options, helper, file_site_histories
|
137
|
-
historical_scan_file = File.join(File.dirname(__FILE__), "#{options[:file_name]}"))
|
142
|
+
def delta_site_report(ticket_repository, options, helper, file_site_histories)
|
138
143
|
# Compares the Scan information from the File && Nexpose.
|
139
144
|
no_processing = true
|
140
145
|
@nexpose_site_histories.each do |site_id, scan_id|
|
141
|
-
# There's no entry in the file, so it's a new site in Nexpose.
|
146
|
+
# There's no entry in the file, so it's either a new site in Nexpose or a new site we have to monitor.
|
142
147
|
if file_site_histories[site_id].nil? || file_site_histories[site_id] == -1
|
143
148
|
full_new_site_report(site_id, ticket_repository, options, helper)
|
144
149
|
no_processing = false
|
@@ -151,18 +156,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
151
156
|
# Done processing, update the CSV to the latest scan info.
|
152
157
|
log_message("Nothing new to process, updating historical CSV file #{options[:file_name]}.") if no_processing
|
153
158
|
log_message("Done processing, updating historical CSV file #{options[:file_name]}.") unless no_processing
|
154
|
-
ticket_repository.save_last_scans(historical_scan_file)
|
155
|
-
log_message('Done updating historical CSV file, service shutting down.')
|
156
159
|
no_processing
|
157
160
|
end
|
158
161
|
|
159
162
|
# There's a new site we haven't seen before.
|
160
163
|
def full_new_site_report(site_id, ticket_repository, options, helper)
|
161
164
|
log_message("New site id: #{site_id} detected. Generating report.")
|
162
|
-
|
165
|
+
new_site_vuln_file = ticket_repository.all_vulns(sites: [site_id], severity: options[:severity])
|
163
166
|
log_message('Report generated, preparing tickets.')
|
164
|
-
|
165
|
-
helper.create_tickets(ticket)
|
167
|
+
ticket_rate_limiter(options, new_site_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch)}, Proc.new {|tickets| helper.create_tickets(tickets)})
|
166
168
|
end
|
167
169
|
|
168
170
|
# There's a new scan with possibly new vulnerabilities.
|
@@ -172,63 +174,143 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
172
174
|
if options[:ticket_mode] == 'I'
|
173
175
|
# I-mode tickets require updating the tickets in the target system.
|
174
176
|
log_message("Scan id for new scan: #{file_site_histories[site_id]}.")
|
175
|
-
|
177
|
+
all_scan_vuln_file = ticket_repository.all_vulns_sites(scan_id: file_site_histories[site_id],
|
176
178
|
site_id: site_id,
|
177
179
|
severity: options[:severity])
|
178
180
|
if helper.respond_to?("prepare_update_tickets") && helper.respond_to?("update_tickets")
|
179
|
-
|
180
|
-
helper.update_tickets(tickets)
|
181
|
+
ticket_rate_limiter(options, all_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_update_tickets(ticket_batch)}, Proc.new {|tickets| helper.update_tickets(tickets)})
|
181
182
|
else
|
182
183
|
log_message("Helper does not implement update methods")
|
183
184
|
fail "Helper using 'I' mode must implement prepare_updates and update_tickets"
|
184
185
|
end
|
185
186
|
else
|
186
187
|
# D-mode tickets require creating new tickets and closing old tickets.
|
187
|
-
|
188
|
+
new_scan_vuln_file = ticket_repository.new_vulns_sites(scan_id: file_site_histories[site_id], site_id: site_id,
|
188
189
|
severity: options[:severity])
|
189
|
-
preparse = CSV.new(
|
190
|
+
preparse = CSV.new(new_scan_vuln_file.path, headers: :first_row)
|
190
191
|
empty_report = preparse.shift.nil?
|
191
192
|
log_message("No new vulnerabilities found in new scan for site: #{site_id}.") if empty_report
|
192
193
|
log_message("New vulnerabilities found in new scan for site #{site_id}, preparing tickets.") unless empty_report
|
193
194
|
unless empty_report
|
194
|
-
|
195
|
-
helper.create_tickets(tickets)
|
195
|
+
ticket_rate_limiter(options, new_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch)}, Proc.new {|tickets| helper.create_tickets(tickets)})
|
196
196
|
end
|
197
197
|
|
198
198
|
if helper.respond_to?("prepare_close_tickets") && helper.respond_to?("close_tickets")
|
199
|
-
|
199
|
+
old_scan_vuln_file = ticket_repository.old_vulns_sites(scan_id: file_site_histories[site_id], site_id: site_id,
|
200
200
|
severity: options[:severity])
|
201
|
-
preparse = CSV.new(
|
201
|
+
preparse = CSV.new(old_scan_vuln_file.path, headers: :first_row, :skip_blanks => true)
|
202
202
|
empty_report = preparse.shift.nil?
|
203
203
|
log_message("No old (closed) vulnerabilities found in new scan for site: #{site_id}.") if empty_report
|
204
204
|
log_message("Old vulnerabilities found in new scan for site #{site_id}, preparing closures.") unless empty_report
|
205
205
|
unless empty_report
|
206
|
-
|
207
|
-
helper.close_tickets(tickets)
|
206
|
+
ticket_rate_limiter(options, old_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_close_tickets(ticket_batch)}, Proc.new {|tickets| helper.close_tickets(tickets)})
|
208
207
|
end
|
209
208
|
else
|
210
209
|
# Create a log message but do not halt execution of the helper if ticket closeing is not
|
211
210
|
# supported to allow legacy code to execute normally.
|
212
|
-
log_message(
|
211
|
+
log_message('Helper does not impelment close methods.')
|
212
|
+
end
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
def ticket_rate_limiter(options, query_results_file, ticket_prepare_method, ticket_send_method)
|
217
|
+
batch_size_max = (options[:batch_size] + 1)
|
218
|
+
log_message("Batching tickets in sizes: #{options[:batch_size]}")
|
219
|
+
|
220
|
+
# Start the batching
|
221
|
+
query_results_file.rewind
|
222
|
+
csv_header = query_results_file.readline
|
223
|
+
ticket_batch = []
|
224
|
+
current_ip = -1
|
225
|
+
current_csv_row = nil
|
226
|
+
|
227
|
+
begin
|
228
|
+
IO.foreach(query_results_file) do |line|
|
229
|
+
ticket_batch << line
|
230
|
+
|
231
|
+
CSV.parse(line.chomp, headers: csv_header) do |row|
|
232
|
+
if current_ip == -1
|
233
|
+
current_ip = row['ip_address'] unless row['ip_address'] == 'ip_address'
|
234
|
+
end
|
235
|
+
current_csv_row = row unless row['ip_address'] == 'ip_address'
|
236
|
+
end
|
237
|
+
|
238
|
+
if ticket_batch.size >= batch_size_max
|
239
|
+
#Batch target reached. Make sure we end with a complete IP address set (all tickets for a single IP in this batch)
|
240
|
+
if(current_ip != current_csv_row['ip_address'])
|
241
|
+
log_message('Batch size reached. Sending tickets.')
|
242
|
+
|
243
|
+
#Move the mismatching line to the next batch
|
244
|
+
line_holder = ticket_batch.pop
|
245
|
+
ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)
|
246
|
+
# Cleanup for the next batch
|
247
|
+
ticket_batch.clear
|
248
|
+
ticket_batch << csv_header
|
249
|
+
ticket_batch << line_holder
|
250
|
+
current_ip = -1
|
251
|
+
end
|
252
|
+
end
|
213
253
|
end
|
254
|
+
ensure
|
255
|
+
log_message('Finished reading report. Sending any remaining tickets and cleaning up file system.')
|
256
|
+
ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)
|
257
|
+
query_results_file.close
|
258
|
+
query_results_file.unlink
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
def ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)
|
263
|
+
#Just the header (no tickets).
|
264
|
+
if ticket_batch.size == 1
|
265
|
+
log_message('Received empty batch. Not sending tickets.')
|
266
|
+
return
|
214
267
|
end
|
268
|
+
|
269
|
+
# Prep the batch of tickets
|
270
|
+
log_message('Creating tickets.')
|
271
|
+
tickets = ticket_prepare_method.call(ticket_batch.join(''))
|
272
|
+
log_message("Generated tickets: #{ticket_batch.size}")
|
273
|
+
# Sent them off
|
274
|
+
log_message('Sending tickets.')
|
275
|
+
ticket_send_method.call(tickets)
|
276
|
+
log_message('Returning for next batch.')
|
215
277
|
end
|
216
278
|
|
279
|
+
|
217
280
|
# Starts the Ticketing Service.
|
218
281
|
def start
|
219
282
|
# Checks if the csv historical file already exists && reads it, otherwise create it && assume first time run.
|
220
283
|
file_site_histories = prepare_historical_data(@ticket_repository, @options)
|
221
|
-
|
222
|
-
|
284
|
+
historical_scan_file = File.join(File.dirname(__FILE__), "#{@options[:file_name]}")
|
285
|
+
# If we didn't specify a site || first time run (no scan history), then it gets all the vulnerabilities.
|
286
|
+
if @options[:sites].empty? || file_site_histories.nil?
|
287
|
+
log_message('Storing current scan state before obtaining all vulnerabilities.')
|
288
|
+
current_scan_state = ticket_repository.load_last_scans(options)
|
289
|
+
|
223
290
|
all_site_report(@ticket_repository, @options, @helper)
|
291
|
+
|
292
|
+
#Generate historical CSV file after completing the fist query.
|
293
|
+
log_message('No historical CSV file found. Generating.')
|
294
|
+
@ticket_repository.save_scans_to_file(historical_scan_file, current_scan_state)
|
295
|
+
log_message('Historical CSV file generated.')
|
224
296
|
else
|
225
297
|
log_message('Obtaining last scan information.')
|
226
|
-
@nexpose_site_histories = @ticket_repository.last_scans
|
298
|
+
@nexpose_site_histories = @ticket_repository.last_scans(@options)
|
299
|
+
|
300
|
+
# Scan states can change during our processing. Store the state we are
|
301
|
+
# about to process and move this to the historical_scan_file if we
|
302
|
+
# successfully process.
|
303
|
+
log_message('Calculated deltas, storing current scan state.')
|
304
|
+
current_scan_state = ticket_repository.load_last_scans(options)
|
305
|
+
|
227
306
|
# Only run if a scan has been ran ever in Nexpose.
|
228
307
|
unless @nexpose_site_histories.empty?
|
229
308
|
delta_site_report(@ticket_repository, @options, @helper, file_site_histories)
|
309
|
+
# Processing completed successfully. Update historical scan file.
|
310
|
+
@ticket_repository.save_scans_to_file(historical_scan_file, current_scan_state)
|
230
311
|
end
|
231
312
|
end
|
313
|
+
log_message('Exiting ticket service.')
|
232
314
|
end
|
233
315
|
end
|
234
316
|
end
|
metadata
CHANGED
@@ -1,41 +1,41 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nexpose_ticketing
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Damian Finol
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-11-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: nexpose
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: 0.6.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 0.6.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: savon
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - ~>
|
31
|
+
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '2.1'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - ~>
|
38
|
+
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '2.1'
|
41
41
|
description: This gem provides a Ruby implementation of different integrations with
|
@@ -67,9 +67,9 @@ files:
|
|
67
67
|
- lib/nexpose_ticketing/helpers/servicenow_helper.rb
|
68
68
|
- lib/nexpose_ticketing/nx_logger.rb
|
69
69
|
- lib/nexpose_ticketing/queries.rb
|
70
|
+
- lib/nexpose_ticketing/report_helper.rb
|
70
71
|
- lib/nexpose_ticketing/ticket_repository.rb
|
71
72
|
- lib/nexpose_ticketing/ticket_service.rb
|
72
|
-
- nexpose_ticketing.gemspec
|
73
73
|
homepage: https://github.com/rapid7/nexpose_ticketing
|
74
74
|
licenses:
|
75
75
|
- BSD
|
@@ -80,17 +80,17 @@ require_paths:
|
|
80
80
|
- lib
|
81
81
|
required_ruby_version: !ruby/object:Gem::Requirement
|
82
82
|
requirements:
|
83
|
-
- -
|
83
|
+
- - ">="
|
84
84
|
- !ruby/object:Gem::Version
|
85
85
|
version: '1.9'
|
86
86
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
87
|
requirements:
|
88
|
-
- -
|
88
|
+
- - ">="
|
89
89
|
- !ruby/object:Gem::Version
|
90
90
|
version: '0'
|
91
91
|
requirements: []
|
92
92
|
rubyforge_project:
|
93
|
-
rubygems_version: 2.
|
93
|
+
rubygems_version: 2.4.4
|
94
94
|
signing_key:
|
95
95
|
specification_version: 4
|
96
96
|
summary: Ruby Nexpose Ticketing Engine.
|
data/nexpose_ticketing.gemspec
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
|
-
Gem::Specification.new do |s|
|
4
|
-
s.name = 'nexpose_ticketing'
|
5
|
-
s.version = '0.2.3'
|
6
|
-
s.homepage = 'https://github.com/rapid7/nexpose_ticketing'
|
7
|
-
s.summary = 'Ruby Nexpose Ticketing Engine.'
|
8
|
-
s.description = 'This gem provides a Ruby implementation of different integrations with ticketing services for Nexpose.'
|
9
|
-
s.license = 'BSD'
|
10
|
-
s.authors = ['Damian Finol']
|
11
|
-
s.email = ['damian_finol@rapid7.com']
|
12
|
-
s.files = Dir['[A-Z]*'] + Dir['lib/**/*'] + Dir['tests/**']
|
13
|
-
s.require_paths = ['lib']
|
14
|
-
s.extra_rdoc_files = ['README.md']
|
15
|
-
s.required_ruby_version = '>= 1.9'
|
16
|
-
s.platform = 'ruby'
|
17
|
-
s.executables = ['nexpose_jira','nexpose_servicenow','nexpose_remedy']
|
18
|
-
s.add_dependency('nexpose', '>= 0.6.0')
|
19
|
-
s.add_dependency('savon', '~> 2.1')
|
20
|
-
end
|