nexpose_ticketing 0.2.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,15 +1,7 @@
1
1
  ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- NjMwMjMyNTVlNjIyMmY2YTNjMzc5ZTVmODFhZDdiZjJjNDI2YzE2OA==
5
- data.tar.gz: !binary |-
6
- YTczODY0MjMzYTg3OTQwMTA1ZDZmMjk1ODdhZmJiZGQ2MGUxOTRjNA==
2
+ SHA1:
3
+ metadata.gz: 8b5571a811b41e5db9bad42f938687414f521da3
4
+ data.tar.gz: 97c1ea6f3932f5b37138f15128f2530eb6bae9f1
7
5
  SHA512:
8
- metadata.gz: !binary |-
9
- ZmUzZDhmMDMxYjBiY2RiOTkyMWNhNDQwMjMwNmM1ZDBlMWM1ZWQxMmVjNDY4
10
- MDRlOWZhYTgyZGY2ZWJmZjhlYmYyYWMzMGE4ZmVjOTZlNjNiYmY5NTEwNmI4
11
- NGQ5OWZhMzExYTAyYjg0Y2M5ZjViNmYwYzg2ZGI2NGIxZGRjNzc=
12
- data.tar.gz: !binary |-
13
- ZmFiNWE4MjAzMzk1N2ExMTFiMWNmZTM4NjIxNjk1MzE2ZjE2OTVmOGVlNjU3
14
- ZGM5MmY0YzZkMmZkZjViMWIwMjFhZTJhM2M3NWIzZmUwNTNiNzk0MjJiMWQy
15
- OWVkZjY2ZmYzNTkxMGJlOWNkMGU4ZjUxNjkwYTYyYTI2N2M0N2Y=
6
+ metadata.gz: a34d4c2ae876184066c674ba39dfa7c2fa792b476c10722d2da0dcd2e4ef8516062f5dfcd8e317a80fc4abfd8001b4b3b8f32cd61d6cf10a1012e561919dd217
7
+ data.tar.gz: a039db4db30f7fdf24a9f56c3851533358085226da163f8a8857abbabce71637e38029c39f48e334926a931abd3c4a0c099058b7e173edc67e94c01d0e181c5c
@@ -1,4 +1,4 @@
1
- #!/usr/bin/env ruby
1
+ #!/usr/bin/ruby
2
2
  require 'yaml'
3
3
  require 'nexpose_ticketing'
4
4
 
@@ -7,11 +7,11 @@
7
7
  :helper_name: ServiceNowHelper
8
8
 
9
9
  # (M) ServiceNow url (currently requires using JSON)
10
- :servicenow_url: https://myinstance.service-now.com/incident.do?JSON
10
+ :servicenow_url: https://your.service-now.com/incident.do?JSON
11
11
  # (M) Username for ServiceNow
12
12
  :username: admin
13
13
  # (M) Password for above username
14
- :password: password
14
+ :password: admin
15
15
  # (M) If 'Y', SSL connections will output to stderr (default is 'N')
16
16
  :verbose_mode: N
17
17
  # (M) Amount of times the helper will follow 301/302 redirections
@@ -8,7 +8,7 @@
8
8
  :logging_enabled: true
9
9
  # Filters the reports to specific sites one per line, leave empty for no site.
10
10
  :sites:
11
- - '1'
11
+ - '7'
12
12
  # Minimum floor severity to report on. Number between 0 and 10.
13
13
  :severity: 8
14
14
  # (M) Name of the report historial file saved in disk.
@@ -17,10 +17,14 @@
17
17
  # 'D' Default IP *-* Vulnerability
18
18
  # 'I' IP address -* Vulnerability
19
19
  :ticket_mode: I
20
+ # Timeout in seconds. The number of seconds the GEM waits for a response from Nexpose before exiting.
21
+ :timeout: 10800
22
+ # Ticket batching. Breaks ticket processing into groups of value size controlling resource utilisation of both systems.
23
+ :batch_size: 300
20
24
  # Nexpose options.
21
25
  :nexpose_data:
22
26
  # (M) Nexpose console hostname.
23
- :nxconsole: 127.0.0.1
27
+ :nxconsole: nexpose-rh.dev.lax.rapid7.com
24
28
  # (M) Nexpose username.
25
29
  :nxuser: nxadmin
26
30
  # (M) Nexpose password.
@@ -176,7 +176,7 @@ class ServiceNowHelper
176
176
  tickets = []
177
177
  current_ip = -1
178
178
  CSV.parse(vulnerability_list.chomp, headers: :first_row) do |row|
179
- if current_ip == -1
179
+ if current_ip == -1
180
180
  current_ip = row['ip_address']
181
181
  @log.log_message("Creating ticket with IP address: #{row['ip_address']}")
182
182
  @ticket = {
@@ -187,24 +187,25 @@ class ServiceNowHelper
187
187
  'urgency' => '1',
188
188
  'short_description' => "#{row['ip_address']} => Vulnerabilities",
189
189
  'work_notes' => "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++
190
- ++ New Vulnerabilities +++++++++++++++++++++++++++++++++++++
190
+ ++ New Vulnerabilities ++++++++++++++++++++++++++++++++++++
191
191
  +++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
192
192
  }
193
193
  end
194
194
  if current_ip == row['ip_address']
195
195
  @ticket['work_notes'] +=
196
- "==========================================
196
+ "\n\n==========================================
197
197
  Summary: #{row['summary']}
198
198
  ----------------------------------------------------------------------------
199
199
  Fix: #{row['fix']}"
200
200
  unless row['url'].nil?
201
201
  @ticket['work_notes'] +=
202
- "----------------------------------------------------------------------------
202
+ "\n----------------------------------------------------------------------------
203
203
  URL: #{row['url']}"
204
204
  end
205
205
  end
206
206
  unless current_ip == row['ip_address']
207
207
  # NXID in the work_notes is the unique identifier used to query incidents to update them.
208
+ @log.log_message("Found new IP address. Finishing ticket with with IP address: #{current_ip} and moving onto IP #{row['ip_address']}")
208
209
  @ticket['work_notes'] += "\nNXID: #{current_ip}"
209
210
  @ticket = @ticket.to_json
210
211
  tickets.push(@ticket)
@@ -213,7 +214,7 @@ class ServiceNowHelper
213
214
  end
214
215
  end
215
216
  # NXID in the work_notes is the unique identifier used to query incidents to update them.
216
- @ticket['work_notes'] += "\nNXID: #{current_ip}"
217
+ @ticket['work_notes'] += "\nNXID: #{current_ip}" unless (@ticket.size == 0)
217
218
  tickets.push(@ticket.to_json) unless @ticket.nil?
218
219
  tickets
219
220
  end
@@ -242,8 +243,12 @@ class ServiceNowHelper
242
243
  ticket_status = row['comparison']
243
244
  @log.log_message("Creating ticket update with IP address: #{row['ip_address']}")
244
245
  @log.log_message("Ticket status #{ticket_status}")
246
+ action = 'update'
247
+ if ticket_status == 'New'
248
+ action = 'insert'
249
+ end
245
250
  @ticket = {
246
- 'sysparm_action' => 'update',
251
+ 'sysparm_action' => action,
247
252
  'sysparm_query' => "work_notesCONTAINSNXID: #{row['ip_address']}",
248
253
  'work_notes' =>
249
254
  "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -283,7 +288,7 @@ class ServiceNowHelper
283
288
  end
284
289
  end
285
290
  # NXID in the work_notes is the unique identifier used to query incidents to update them.
286
- @ticket['work_notes'] += "\nNXID: #{current_ip}"
291
+ @ticket['work_notes'] += "\nNXID: #{current_ip}" unless (@ticket.size == 0)
287
292
  tickets.push(@ticket.to_json) unless @ticket.nil?
288
293
  tickets
289
294
  end
@@ -19,7 +19,7 @@ module NexposeTicketing
19
19
  # |url| |summary| |fix|
20
20
  #
21
21
  def self.all_new_vulns
22
- "SELECT subs.asset_id, da.ip_address, subs.current_scan, subs.vulnerability_id, davs.solution_id, ds.nexpose_id,
22
+ "SELECT DISTINCT on (da.ip_address, davs.solution_id) subs.asset_id, da.ip_address, subs.current_scan, subs.vulnerability_id, davs.solution_id, ds.nexpose_id,
23
23
  ds.url,proofAsText(ds.summary) as summary, proofAsText(ds.fix) as fix
24
24
  FROM (SELECT fasv.asset_id, fasv.vulnerability_id, s.current_scan
25
25
  FROM fact_asset_scan_vulnerability_finding fasv
@@ -28,14 +28,14 @@ module NexposeTicketing
28
28
  SELECT asset_id, previousScan(asset_id) AS baseline_scan, lastScan(asset_id) AS current_scan
29
29
  FROM dim_asset) s
30
30
  ON s.asset_id = fasv.asset_id AND (fasv.scan_id = s.baseline_scan OR fasv.scan_id = s.current_scan)
31
- GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan
32
- HAVING baselineComparison(fasv.scan_id, current_scan) = 'New'
31
+ GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan, fasv.scan_id
32
+ HAVING NOT baselineComparison(fasv.scan_id, current_scan) = 'Old'
33
33
  ) subs
34
34
  JOIN dim_asset_vulnerability_solution davs USING (vulnerability_id)
35
35
  JOIN dim_solution ds USING (solution_id)
36
36
  JOIN dim_asset da ON subs.asset_id = da.asset_id
37
- ORDER BY da.ip_address"
38
- end
37
+ ORDER BY da.ip_address, davs.solution_id"
38
+ end
39
39
 
40
40
  # Gets all new vulnerabilities happening after a reported scan id.
41
41
  #
@@ -0,0 +1,76 @@
1
+ require 'nexpose'
2
+ require 'securerandom'
3
+ include Nexpose
4
+
5
+ module NexposeReportHelper
6
+ class ReportOps
7
+
8
+ def initialize(nsc, timeout)
9
+ @timeout = timeout
10
+ @nsc = nsc
11
+ end
12
+
13
+ def generate_sql_report_config
14
+ random_name = "Nexpose-ticketing-Temp-#{SecureRandom.uuid}"
15
+ Nexpose::ReportConfig.new(random_name, nil, 'sql')
16
+ end
17
+
18
+ def save_generate_cleanup_report_config(report_config)
19
+ report_id = report_config.save(@nsc, false)
20
+ @nsc.generate_report(report_id, true)
21
+ wait_for_report(report_id)
22
+ report_details = @nsc.last_report(report_id)
23
+ file = Tempfile.new("#{report_id}")
24
+ file.write(@nsc.download(report_details.uri))
25
+ #Got the report, cleanup server-side
26
+ @nsc.delete_report_config(report_id)
27
+ file
28
+ end
29
+
30
+ # Wait for report generation to complete.
31
+ #
32
+ # @param [Fixnum] id Report configuration ID of the report waiting to generate.
33
+ #
34
+ def wait_for_report(id)
35
+ wait_until(:fail_on_exceptions => TRUE, :on_timeout => "Report generation timed out. Status: #{r = @nsc.last_report(id); r ? r.status : 'unknown'}") {
36
+ if %w(Failed Aborted Unknown).include?(@nsc.last_report(id).status)
37
+ raise "Report failed to generate! Status <#{@nsc.last_report(id).status}>"
38
+ end
39
+ @nsc.last_report(id).status == 'Generated'
40
+ }
41
+ end
42
+
43
+ # Wait for a given block to evaluate to true.
44
+ #
45
+ # The following flags are accepted as arguments:
46
+ # :timeout Number of seconds to wait before timing out. Defaults to 90.
47
+ # :polling_interval Number of seconds to wait between checking block again.
48
+ # Defaults to 5.
49
+ # :fail_on_exceptions Whether to fail fast. Defaults to off.
50
+ # :on_timeout Message to raise with the exception when a timeout occurs.
51
+ #
52
+ # Example usage:
53
+ # wait_until { site.id > 0 }
54
+ # wait_until(:timeout => 30, :polling_interval => 0.5) { 1 == 2 }
55
+ # wait_until(:on_timeout => 'Unable to confirm scan integration.') { console.sites.find { |site| site[:site_id] == site_id.to_i }[:risk_score] > 0.0 }
56
+ #
57
+ def wait_until(options = {})
58
+ polling_interval = 90
59
+ time_limit = Time.now + @timeout
60
+ loop do
61
+ begin
62
+ val = yield
63
+ return val if val
64
+ rescue Exception => error
65
+ raise error if options[:fail_on_exceptions]
66
+ end
67
+ if Time.now >= time_limit
68
+ raise options[:on_timeout] if options[:on_timeout]
69
+ error ||= 'Timed out waiting for condition.'
70
+ raise error
71
+ end
72
+ sleep polling_interval
73
+ end
74
+ end
75
+ end
76
+ end
@@ -4,10 +4,28 @@ module NexposeTicketing
4
4
  require 'csv'
5
5
  require 'nexpose'
6
6
  require 'nexpose_ticketing/queries'
7
+ require 'nexpose_ticketing/report_helper'
8
+
9
+ @timeout = 10800
10
+
11
+ def initialize(options = nil)
12
+ @timeout = options[:timeout]
13
+ end
7
14
 
8
15
  def nexpose_login(nexpose_data)
9
16
  @nsc = Nexpose::Connection.new(nexpose_data[:nxconsole], nexpose_data[:nxuser], nexpose_data[:nxpasswd])
10
17
  @nsc.login
18
+ #After login, create the report helper
19
+ @report_helper = NexposeReportHelper::ReportOps.new(@nsc, @timeout)
20
+ end
21
+
22
+ # Returns an array of all sites in the users environment.
23
+ #
24
+ # * *Returns* :
25
+ # - An array of Nexpose::SiteSummary objects.
26
+ #
27
+ def all_site_details()
28
+ @nsc.sites
11
29
  end
12
30
 
13
31
  # Reads the site scan history from disk.
@@ -31,28 +49,56 @@ module NexposeTicketing
31
49
  # * *Args* :
32
50
  # - +csv_file_name+ - CSV File name.
33
51
  #
34
- def save_last_scans(csv_file_name, saved_file = nil, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
52
+ def save_last_scans(csv_file_name, options = {}, saved_file = nil)
53
+ current_scan_state = load_last_scans(options)
54
+ save_scans_to_file(csv_file_name, current_scan_state, saved_file)
55
+ end
56
+
57
+ # Loads the last scan info to memory.
58
+ #
59
+ # * *Args* :
60
+ # - +csv_file_name+ - CSV File name.
61
+ #
62
+ def load_last_scans(options = {}, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
63
+ sites = Array(options[:sites])
35
64
  report_config.add_filter('version', '1.2.0')
36
65
  report_config.add_filter('query', Queries.last_scans)
37
- report_output = report_config.generate(@nsc)
66
+
67
+ report_output = report_config.generate(@nsc, @timeout)
38
68
  csv_output = CSV.parse(report_output.chomp, headers: :first_row)
39
- saved_file.open(csv_file_name, 'w') { |file| file.puts(csv_output) } unless saved_file.nil?
69
+
70
+ #We only care about sites we are monitoring.
71
+ trimmed_csv = []
72
+ trimmed_csv << report_output.lines.first
73
+ csv_output.each do |row|
74
+ if sites.include? row.to_s[0]
75
+ trimmed_csv << row
76
+ end
77
+ end
78
+ trimmed_csv
79
+ end
80
+
81
+ # Saves CSV scan information to disk
82
+ #
83
+ # * *Args* :
84
+ # - +csv_file_name+ - CSV File name.
85
+ #
86
+ def save_scans_to_file(csv_file_name, trimmed_csv, saved_file = nil)
87
+ saved_file.open(csv_file_name, 'w') { |file| file.puts(trimmed_csv) } unless saved_file.nil?
40
88
  if saved_file.nil?
41
- File.open(csv_file_name, 'w') { |file| file.puts(csv_output) }
89
+ File.open(csv_file_name, 'w') { |file| file.puts(trimmed_csv) }
42
90
  end
43
91
  end
44
92
 
45
- # Gets the last scan information from nexpose.
93
+ # Gets the last scan information from nexpose sans the CSV headers.
46
94
  #
47
95
  # * *Returns* :
48
96
  # - A hash with site_ids => last_scan_id
49
97
  #
50
- def last_scans(report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
51
- report_config.add_filter('version', '1.2.0')
52
- report_config.add_filter('query', Queries.last_scans)
53
- report_output = report_config.generate(@nsc).chomp
98
+ def last_scans(options = {})
54
99
  nexpose_sites = Hash.new(-1)
55
- CSV.parse(report_output, headers: :first_row) do |row|
100
+ trimmed_csv = load_last_scans(options)
101
+ trimmed_csv.drop(1).each do |row|
56
102
  nexpose_sites[row['site_id']] = row['last_scan_id'].to_i
57
103
  end
58
104
  nexpose_sites
@@ -62,23 +108,29 @@ module NexposeTicketing
62
108
  #
63
109
  # * *Args* :
64
110
  # - +site_options+ - A Hash with site(s) and severity level.
111
+ # - +site_to_query+ - Override for user-configured site options
65
112
  #
66
113
  # * *Returns* :
67
114
  # - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
68
115
  # |url| |summary| |fix|
69
116
  #
70
- def all_vulns(site_options = {}, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
71
- sites = Array(site_options[:sites])
72
- severity = site_options[:severity].nil? ? 0 : site_options[:severity]
117
+ def all_vulns(options = {}, site_override = nil)
118
+ if site_override.nil?
119
+ sites = Array(options[:sites])
120
+ else
121
+ sites = site_override
122
+ end
123
+ report_config = @report_helper.generate_sql_report_config()
124
+ severity = options[:severity].nil? ? 0 : options[:severity]
73
125
  report_config.add_filter('version', '1.2.0')
74
126
  report_config.add_filter('query', Queries.all_new_vulns)
75
127
  unless sites.empty?
76
- sites.each do |site_id|
128
+ Array(sites).each do |site_id|
77
129
  report_config.add_filter('site', site_id)
78
130
  end
79
131
  end
80
132
  report_config.add_filter('vuln-severity', severity)
81
- report_config.generate(@nsc)
133
+ @report_helper.save_generate_cleanup_report_config(report_config)
82
134
  end
83
135
 
84
136
  # Gets the new vulns from base scan reported_scan_id and the newest / latest scan from a site.
@@ -90,7 +142,8 @@ module NexposeTicketing
90
142
  # - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
91
143
  # |url| |summary| |fix|
92
144
  #
93
- def new_vulns_sites(site_options = {}, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
145
+ def new_vulns_sites(site_options = {})
146
+ report_config = @report_helper.generate_sql_report_config()
94
147
  site = site_options[:site_id]
95
148
  reported_scan_id = site_options[:scan_id]
96
149
  fail 'Site cannot be null or empty' if site.nil? || reported_scan_id.nil?
@@ -99,7 +152,7 @@ module NexposeTicketing
99
152
  report_config.add_filter('query', Queries.new_vulns_since_scan(reported_scan_id))
100
153
  report_config.add_filter('site', site)
101
154
  report_config.add_filter('vuln-severity', severity)
102
- report_config.generate(@nsc)
155
+ @report_helper.save_generate_cleanup_report_config(report_config)
103
156
  end
104
157
 
105
158
  # Gets the old vulns from base scan reported_scan_id and the newest / latest scan from a site.
@@ -111,7 +164,8 @@ module NexposeTicketing
111
164
  # - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
112
165
  # |url| |summary| |fix|
113
166
  #
114
- def old_vulns_sites(site_options = {}, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
167
+ def old_vulns_sites(site_options = {})
168
+ report_config = @report_helper.generate_sql_report_config()
115
169
  site = site_options[:site_id]
116
170
  reported_scan_id = site_options[:scan_id]
117
171
  fail 'Site cannot be null or empty' if site.nil? || reported_scan_id.nil?
@@ -120,7 +174,7 @@ module NexposeTicketing
120
174
  report_config.add_filter('query', Queries.old_vulns_since_scan(reported_scan_id))
121
175
  report_config.add_filter('site', site)
122
176
  report_config.add_filter('vuln-severity', severity)
123
- report_config.generate(@nsc)
177
+ @report_helper.save_generate_cleanup_report_config(report_config)
124
178
  end
125
179
 
126
180
  # Gets all vulns from base scan reported_scan_id and the newest / latest scan from a site. This is
@@ -133,7 +187,8 @@ module NexposeTicketing
133
187
  # - Returns CSV |asset_id| |ip_address| |current_scan| |vulnerability_id| |solution_id| |nexpose_id|
134
188
  # |url| |summary| |fix| |comparison|
135
189
  #
136
- def all_vulns_sites(site_options = {}, report_config = Nexpose::AdhocReportConfig.new(nil, 'sql'))
190
+ def all_vulns_sites(site_options = {})
191
+ report_config = @report_helper.generate_sql_report_config()
137
192
  site = site_options[:site_id]
138
193
  reported_scan_id = site_options[:scan_id]
139
194
  fail 'Site cannot be null or empty' if site.nil? || reported_scan_id.nil?
@@ -142,7 +197,7 @@ module NexposeTicketing
142
197
  report_config.add_filter('query', Queries.all_vulns_since_scan(reported_scan_id))
143
198
  report_config.add_filter('site', site)
144
199
  report_config.add_filter('vuln-severity', severity)
145
- report_config.generate(@nsc)
200
+ @report_helper.save_generate_cleanup_report_config(report_config)
146
201
  end
147
202
  end
148
203
  end
@@ -81,9 +81,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
81
81
  end
82
82
  log_message("Enabling helper: #{@helper_data[:helper_name]}.")
83
83
  @helper = eval(@helper_data[:helper_name]).new(@helper_data, @options)
84
- @ticket_repository = NexposeTicketing::TicketRepository.new
84
+
85
+ log_message("Creating ticketing repository with timeout value: #{@options[:timeout]}.")
86
+ @ticket_repository = NexposeTicketing::TicketRepository.new(options)
85
87
  @ticket_repository.nexpose_login(@nexpose_data)
86
- @first_time = false
87
88
  end
88
89
 
89
90
  def setup_logging(enabled = false)
@@ -109,36 +110,40 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
109
110
  log_message("Reading historical CSV file: #{historical_scan_file}.")
110
111
  file_site_histories = ticket_repository.read_last_scans(historical_scan_file)
111
112
  else
112
- log_message('No historical CSV file found. Generating.')
113
- ticket_repository.save_last_scans(historical_scan_file)
114
- log_message('Historical CSV file generated.')
115
- file_site_histories = ticket_repository.read_last_scans(historical_scan_file)
116
- @first_time = true
113
+ file_site_histories = nil
117
114
  end
118
115
  file_site_histories
119
116
  end
120
117
 
121
118
  # Generates a full site(s) report ticket(s).
122
- def all_site_report(ticket_repository, options, helper,
123
- historical_scan_file = File.join(File.dirname(__FILE__), "#{options[:file_name]}"))
124
- log_message('First time run, generating full vulnerability report.') if @first_time
125
- log_message('No site(s) specified, generating full vulnerability report.') if options[:sites].empty?
126
- all_delta_vulns = ticket_repository.all_vulns(severity: options[:severity])
127
- log_message('Preparing tickets.')
128
- tickets = helper.prepare_create_tickets(all_delta_vulns)
129
- helper.create_tickets(tickets)
130
- log_message("Done processing, updating historical CSV file #{historical_scan_file}.")
131
- ticket_repository.save_last_scans(historical_scan_file)
132
- log_message('Done updating historical CSV file, service shutting down.')
119
+ def all_site_report(ticket_repository, options, helper)
120
+
121
+ sites_to_query = Array.new
122
+ if options[:sites].empty?
123
+ log_message('No site(s) specified, generating full vulnerability report.')
124
+ @ticket_repository.all_site_details.each { |site| sites_to_query << site.id }
125
+ else
126
+ log_message('Generating full vulnerability report on user entered sites.')
127
+ sites_to_query = Array(options[:sites])
128
+ end
129
+
130
+ log_message("Generating full vulnerability report on the following sites: #{sites_to_query.join(', ')}")
131
+
132
+ sites_to_query.each { |site|
133
+ log_message("Running full vulnerability report on site #{site}")
134
+ all_vulns_file = ticket_repository.all_vulns(options, site)
135
+ log_message('Preparing tickets.')
136
+ ticket_rate_limiter(options, all_vulns_file, Proc.new { |ticket_batch| helper.prepare_create_tickets(ticket_batch) }, Proc.new { |tickets| helper.create_tickets(tickets) })
137
+ }
138
+ log_message('Finished process all vulnerabilities.')
133
139
  end
134
140
 
135
141
  # There's possibly a new scan with new data.
136
- def delta_site_report(ticket_repository, options, helper, file_site_histories,
137
- historical_scan_file = File.join(File.dirname(__FILE__), "#{options[:file_name]}"))
142
+ def delta_site_report(ticket_repository, options, helper, file_site_histories)
138
143
  # Compares the Scan information from the File && Nexpose.
139
144
  no_processing = true
140
145
  @nexpose_site_histories.each do |site_id, scan_id|
141
- # There's no entry in the file, so it's a new site in Nexpose.
146
+ # There's no entry in the file, so it's either a new site in Nexpose or a new site we have to monitor.
142
147
  if file_site_histories[site_id].nil? || file_site_histories[site_id] == -1
143
148
  full_new_site_report(site_id, ticket_repository, options, helper)
144
149
  no_processing = false
@@ -151,18 +156,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
151
156
  # Done processing, update the CSV to the latest scan info.
152
157
  log_message("Nothing new to process, updating historical CSV file #{options[:file_name]}.") if no_processing
153
158
  log_message("Done processing, updating historical CSV file #{options[:file_name]}.") unless no_processing
154
- ticket_repository.save_last_scans(historical_scan_file)
155
- log_message('Done updating historical CSV file, service shutting down.')
156
159
  no_processing
157
160
  end
158
161
 
159
162
  # There's a new site we haven't seen before.
160
163
  def full_new_site_report(site_id, ticket_repository, options, helper)
161
164
  log_message("New site id: #{site_id} detected. Generating report.")
162
- new_site_vuln = ticket_repository.all_vulns(sites: [site_id], severity: options[:severity])
165
+ new_site_vuln_file = ticket_repository.all_vulns(sites: [site_id], severity: options[:severity])
163
166
  log_message('Report generated, preparing tickets.')
164
- ticket = helper.prepare_create_tickets(new_site_vuln)
165
- helper.create_tickets(ticket)
167
+ ticket_rate_limiter(options, new_site_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch)}, Proc.new {|tickets| helper.create_tickets(tickets)})
166
168
  end
167
169
 
168
170
  # There's a new scan with possibly new vulnerabilities.
@@ -172,63 +174,143 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
172
174
  if options[:ticket_mode] == 'I'
173
175
  # I-mode tickets require updating the tickets in the target system.
174
176
  log_message("Scan id for new scan: #{file_site_histories[site_id]}.")
175
- all_scan_vuln = ticket_repository.all_vulns_sites(scan_id: file_site_histories[site_id],
177
+ all_scan_vuln_file = ticket_repository.all_vulns_sites(scan_id: file_site_histories[site_id],
176
178
  site_id: site_id,
177
179
  severity: options[:severity])
178
180
  if helper.respond_to?("prepare_update_tickets") && helper.respond_to?("update_tickets")
179
- tickets = helper.prepare_update_tickets(all_scan_vuln)
180
- helper.update_tickets(tickets)
181
+ ticket_rate_limiter(options, all_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_update_tickets(ticket_batch)}, Proc.new {|tickets| helper.update_tickets(tickets)})
181
182
  else
182
183
  log_message("Helper does not implement update methods")
183
184
  fail "Helper using 'I' mode must implement prepare_updates and update_tickets"
184
185
  end
185
186
  else
186
187
  # D-mode tickets require creating new tickets and closing old tickets.
187
- new_scan_vuln = ticket_repository.new_vulns_sites(scan_id: file_site_histories[site_id], site_id: site_id,
188
+ new_scan_vuln_file = ticket_repository.new_vulns_sites(scan_id: file_site_histories[site_id], site_id: site_id,
188
189
  severity: options[:severity])
189
- preparse = CSV.new(new_scan_vuln.chomp, headers: :first_row)
190
+ preparse = CSV.new(new_scan_vuln_file.path, headers: :first_row)
190
191
  empty_report = preparse.shift.nil?
191
192
  log_message("No new vulnerabilities found in new scan for site: #{site_id}.") if empty_report
192
193
  log_message("New vulnerabilities found in new scan for site #{site_id}, preparing tickets.") unless empty_report
193
194
  unless empty_report
194
- tickets = helper.prepare_create_tickets(new_scan_vuln)
195
- helper.create_tickets(tickets)
195
+ ticket_rate_limiter(options, new_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch)}, Proc.new {|tickets| helper.create_tickets(tickets)})
196
196
  end
197
197
 
198
198
  if helper.respond_to?("prepare_close_tickets") && helper.respond_to?("close_tickets")
199
- old_scan_vuln = ticket_repository.old_vulns_sites(scan_id: file_site_histories[site_id], site_id: site_id,
199
+ old_scan_vuln_file = ticket_repository.old_vulns_sites(scan_id: file_site_histories[site_id], site_id: site_id,
200
200
  severity: options[:severity])
201
- preparse = CSV.new(old_scan_vuln.chomp, headers: :first_row)
201
+ preparse = CSV.new(old_scan_vuln_file.path, headers: :first_row, :skip_blanks => true)
202
202
  empty_report = preparse.shift.nil?
203
203
  log_message("No old (closed) vulnerabilities found in new scan for site: #{site_id}.") if empty_report
204
204
  log_message("Old vulnerabilities found in new scan for site #{site_id}, preparing closures.") unless empty_report
205
205
  unless empty_report
206
- tickets = helper.prepare_close_tickets(old_scan_vuln)
207
- helper.close_tickets(tickets)
206
+ ticket_rate_limiter(options, old_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_close_tickets(ticket_batch)}, Proc.new {|tickets| helper.close_tickets(tickets)})
208
207
  end
209
208
  else
210
209
  # Create a log message but do not halt execution of the helper if ticket closeing is not
211
210
  # supported to allow legacy code to execute normally.
212
- log_message("Helper does not impelment close methods.")
211
+ log_message('Helper does not impelment close methods.')
212
+ end
213
+ end
214
+ end
215
+
216
+ def ticket_rate_limiter(options, query_results_file, ticket_prepare_method, ticket_send_method)
217
+ batch_size_max = (options[:batch_size] + 1)
218
+ log_message("Batching tickets in sizes: #{options[:batch_size]}")
219
+
220
+ # Start the batching
221
+ query_results_file.rewind
222
+ csv_header = query_results_file.readline
223
+ ticket_batch = []
224
+ current_ip = -1
225
+ current_csv_row = nil
226
+
227
+ begin
228
+ IO.foreach(query_results_file) do |line|
229
+ ticket_batch << line
230
+
231
+ CSV.parse(line.chomp, headers: csv_header) do |row|
232
+ if current_ip == -1
233
+ current_ip = row['ip_address'] unless row['ip_address'] == 'ip_address'
234
+ end
235
+ current_csv_row = row unless row['ip_address'] == 'ip_address'
236
+ end
237
+
238
+ if ticket_batch.size >= batch_size_max
239
+ #Batch target reached. Make sure we end with a complete IP address set (all tickets for a single IP in this batch)
240
+ if(current_ip != current_csv_row['ip_address'])
241
+ log_message('Batch size reached. Sending tickets.')
242
+
243
+ #Move the mismatching line to the next batch
244
+ line_holder = ticket_batch.pop
245
+ ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)
246
+ # Cleanup for the next batch
247
+ ticket_batch.clear
248
+ ticket_batch << csv_header
249
+ ticket_batch << line_holder
250
+ current_ip = -1
251
+ end
252
+ end
213
253
  end
254
+ ensure
255
+ log_message('Finished reading report. Sending any remaining tickets and cleaning up file system.')
256
+ ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)
257
+ query_results_file.close
258
+ query_results_file.unlink
259
+ end
260
+ end
261
+
262
+ def ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)
263
+ #Just the header (no tickets).
264
+ if ticket_batch.size == 1
265
+ log_message('Received empty batch. Not sending tickets.')
266
+ return
214
267
  end
268
+
269
+ # Prep the batch of tickets
270
+ log_message('Creating tickets.')
271
+ tickets = ticket_prepare_method.call(ticket_batch.join(''))
272
+ log_message("Generated tickets: #{ticket_batch.size}")
273
+ # Sent them off
274
+ log_message('Sending tickets.')
275
+ ticket_send_method.call(tickets)
276
+ log_message('Returning for next batch.')
215
277
  end
216
278
 
279
+
217
280
  # Starts the Ticketing Service.
218
281
  def start
219
282
  # Checks if the csv historical file already exists && reads it, otherwise create it && assume first time run.
220
283
  file_site_histories = prepare_historical_data(@ticket_repository, @options)
221
- # If we didn't specify a site || first time run, then it gets all the vulnerabilities.
222
- if @options[:sites].empty? || @first_time
284
+ historical_scan_file = File.join(File.dirname(__FILE__), "#{@options[:file_name]}")
285
+ # If we didn't specify a site || first time run (no scan history), then it gets all the vulnerabilities.
286
+ if @options[:sites].empty? || file_site_histories.nil?
287
+ log_message('Storing current scan state before obtaining all vulnerabilities.')
288
+ current_scan_state = ticket_repository.load_last_scans(options)
289
+
223
290
  all_site_report(@ticket_repository, @options, @helper)
291
+
292
+ #Generate historical CSV file after completing the fist query.
293
+ log_message('No historical CSV file found. Generating.')
294
+ @ticket_repository.save_scans_to_file(historical_scan_file, current_scan_state)
295
+ log_message('Historical CSV file generated.')
224
296
  else
225
297
  log_message('Obtaining last scan information.')
226
- @nexpose_site_histories = @ticket_repository.last_scans
298
+ @nexpose_site_histories = @ticket_repository.last_scans(@options)
299
+
300
+ # Scan states can change during our processing. Store the state we are
301
+ # about to process and move this to the historical_scan_file if we
302
+ # successfully process.
303
+ log_message('Calculated deltas, storing current scan state.')
304
+ current_scan_state = ticket_repository.load_last_scans(options)
305
+
227
306
  # Only run if a scan has been ran ever in Nexpose.
228
307
  unless @nexpose_site_histories.empty?
229
308
  delta_site_report(@ticket_repository, @options, @helper, file_site_histories)
309
+ # Processing completed successfully. Update historical scan file.
310
+ @ticket_repository.save_scans_to_file(historical_scan_file, current_scan_state)
230
311
  end
231
312
  end
313
+ log_message('Exiting ticket service.')
232
314
  end
233
315
  end
234
316
  end
metadata CHANGED
@@ -1,41 +1,41 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: nexpose_ticketing
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.3
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Damian Finol
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-07-08 00:00:00.000000000 Z
11
+ date: 2014-11-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: nexpose
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ! '>='
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
19
  version: 0.6.0
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ! '>='
24
+ - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: 0.6.0
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: savon
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ~>
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
33
  version: '2.1'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ~>
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '2.1'
41
41
  description: This gem provides a Ruby implementation of different integrations with
@@ -67,9 +67,9 @@ files:
67
67
  - lib/nexpose_ticketing/helpers/servicenow_helper.rb
68
68
  - lib/nexpose_ticketing/nx_logger.rb
69
69
  - lib/nexpose_ticketing/queries.rb
70
+ - lib/nexpose_ticketing/report_helper.rb
70
71
  - lib/nexpose_ticketing/ticket_repository.rb
71
72
  - lib/nexpose_ticketing/ticket_service.rb
72
- - nexpose_ticketing.gemspec
73
73
  homepage: https://github.com/rapid7/nexpose_ticketing
74
74
  licenses:
75
75
  - BSD
@@ -80,17 +80,17 @@ require_paths:
80
80
  - lib
81
81
  required_ruby_version: !ruby/object:Gem::Requirement
82
82
  requirements:
83
- - - ! '>='
83
+ - - ">="
84
84
  - !ruby/object:Gem::Version
85
85
  version: '1.9'
86
86
  required_rubygems_version: !ruby/object:Gem::Requirement
87
87
  requirements:
88
- - - ! '>='
88
+ - - ">="
89
89
  - !ruby/object:Gem::Version
90
90
  version: '0'
91
91
  requirements: []
92
92
  rubyforge_project:
93
- rubygems_version: 2.2.2
93
+ rubygems_version: 2.4.4
94
94
  signing_key:
95
95
  specification_version: 4
96
96
  summary: Ruby Nexpose Ticketing Engine.
@@ -1,20 +0,0 @@
1
- # encoding: utf-8
2
-
3
- Gem::Specification.new do |s|
4
- s.name = 'nexpose_ticketing'
5
- s.version = '0.2.3'
6
- s.homepage = 'https://github.com/rapid7/nexpose_ticketing'
7
- s.summary = 'Ruby Nexpose Ticketing Engine.'
8
- s.description = 'This gem provides a Ruby implementation of different integrations with ticketing services for Nexpose.'
9
- s.license = 'BSD'
10
- s.authors = ['Damian Finol']
11
- s.email = ['damian_finol@rapid7.com']
12
- s.files = Dir['[A-Z]*'] + Dir['lib/**/*'] + Dir['tests/**']
13
- s.require_paths = ['lib']
14
- s.extra_rdoc_files = ['README.md']
15
- s.required_ruby_version = '>= 1.9'
16
- s.platform = 'ruby'
17
- s.executables = ['nexpose_jira','nexpose_servicenow','nexpose_remedy']
18
- s.add_dependency('nexpose', '>= 0.6.0')
19
- s.add_dependency('savon', '~> 2.1')
20
- end