nexpose_servicenow 0.4.24 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,12 +7,12 @@ module NexposeServiceNow
7
7
  class NexposeHelper
8
8
  def initialize(url, port, username, password)
9
9
  @log = NexposeServiceNow::NxLogger.instance
10
- @url = url
11
- @port = port
10
+ @url = url
11
+ @port = port
12
12
  @username = username
13
13
  @password = password
14
14
 
15
- @nsc = connect(username, password)
15
+ @nsc = connect(username, password)
16
16
 
17
17
  @timeout = 7200
18
18
  end
@@ -22,7 +22,7 @@ module NexposeServiceNow
22
22
  return [ id: -1, report_name: get_report_name(query_name) ]
23
23
  end
24
24
 
25
- ids.map { |id| { id: id, report_name: get_report_name(query_name, id) }}
25
+ ids.map { |id| { id: id, report_name: get_report_name(query_name, id) } }
26
26
  end
27
27
 
28
28
  def self.get_report_name(query_name, id=nil)
@@ -37,36 +37,60 @@ module NexposeServiceNow
37
37
  end
38
38
 
39
39
  def create_report(query_name, ids, id_type, output_dir, query_options={})
40
- output_dir = File.expand_path(output_dir.to_s)
40
+ output_dir = File.expand_path(output_dir.to_s)
41
41
 
42
42
  #A single report doesn't use site filters
43
43
  ids = [-1] unless Queries.multiple_reports?(query_name)
44
44
 
45
+ #If running latest_scans, send up information
46
+ if query_name == 'latest_scans'
47
+ @log.on_connect(@url, @port, @nsc.session_id, '{}')
48
+ end
49
+
45
50
  reports = []
46
51
  ids.each do |id|
47
52
  report_name = self.class.get_report_name(query_name, id)
48
53
  clean_up_reports(report_name)
49
54
 
50
- delta_options = create_query_options(query_options, id)
51
- delta_options['site_id'] = id
55
+ # Should use the value from historical_data.rb
56
+ default_delta = if id_type.to_s == 'asset_group'
57
+ '1985-01-01 00:00:00'
58
+ elsif id_type.to_s == 'site'
59
+ 0
60
+ end
61
+
62
+ delta_options = create_query_options(query_options, id, default_delta)
63
+
64
+ # TODO: Refactor this into 'create_query_options'
65
+ delta_options[:site_id] = id
66
+ delta_options[:id_type] = id_type.to_s
67
+ delta_options[:filters] = query_options[:filters] || {}
68
+
69
+ min_cvss = if delta_options[:filters][:cvss].nil?
70
+ 0
71
+ else
72
+ delta_options[:filters][:cvss].first
73
+ end
52
74
 
53
75
  query = Queries.send(query_name, delta_options)
54
- report_id = generate_config(query, report_name, [id], id_type)
76
+
77
+ report_id = generate_config(query, report_name, [id], id_type, min_cvss)
55
78
 
56
79
  run_report(report_id)
57
80
  reports << save_report(report_name, report_id, output_dir)
58
81
  end
59
82
 
60
- return reports
83
+ reports
61
84
  end
62
85
 
63
- def create_query_options(query_options, nexpose_id=nil)
86
+ def create_query_options(query_options, nexpose_id=nil, default=0)
64
87
  options = {}
65
88
  options[:vuln_query_date] = query_options[:vuln_query_date]
66
89
 
67
- return options if nexpose_id == nil || nexpose_id == -1
68
- return 0 if query_options[:last_scans].empty?
69
- options[:last_scan_id] = "#{query_options[:last_scans][nexpose_id] || 0}"
90
+ return options if nexpose_id.nil? || nexpose_id == -1
91
+ return 0 if query_options[:delta_values].empty?
92
+
93
+ options[:delta] = "#{query_options[:delta_values][nexpose_id] || default}"
70
94
 
71
95
  @log.log_message("Query options: #{options}")
72
96
 
@@ -74,23 +98,23 @@ module NexposeServiceNow
74
98
  end
75
99
 
76
100
  def clean_up_reports(report_name)
77
- #log 'Deleting existing report...'
78
101
  reports = @nsc.list_reports
79
- reports.select! { |r| r.name.start_with? report_name }
80
- reports.each do |r|
81
- @nsc.delete_report_config(r.config_id)
82
- end
83
- end
102
+ reports.select! { |r| r.name.start_with? report_name }
103
+ reports.each { |r| @nsc.delete_report_config(r.config_id) }
104
+ end
84
105
 
85
- def generate_config(query, report_name, ids, id_type)
106
+ def generate_config(query, report_name, ids, id_type, min_severity=0)
86
107
  @nsc = connect(@username, @password)
87
- @log.log_message "Generating query config with name #{report_name}..."
88
- report_config = Nexpose::ReportConfig.new(report_name, nil, 'sql')
108
+ @log.log_message "Generating report config with name #{report_name}..."
109
+ report_config = Nexpose::ReportConfig.new(report_name, nil, 'sql')
89
110
  report_config.add_filter('version', '2.0.1')
90
111
  report_config.add_filter('query', query)
91
112
 
113
+ id_type = id_type.to_s.split('_').last
114
+ ids.each { |id| report_config.add_filter(id_type, id) unless id == -1 }
115
+
116
+
92
117
  @log.log_message "Saving report config #{report_name}..."
93
- ids.each { |id| report_config.add_filter(id_type, id) if id != -1 }
94
118
  report_id = report_config.save(@nsc, false)
95
119
  @log.log_message "Report #{report_name} saved"
96
120
 
@@ -98,13 +122,13 @@ module NexposeServiceNow
98
122
  end
99
123
 
100
124
  def run_report(report_id)
101
- @log.log_message "Running report #{report_id}..."
125
+ @log.log_message "Running report #{report_id}..."
102
126
  @nsc.generate_report(report_id, false)
103
127
  wait_for_report(report_id)
104
128
  end
105
129
 
106
130
  def wait_for_report(id)
107
- wait_until(:fail_on_exceptions => TRUE, :on_timeout => "Report generation timed out. Status: #{r = @nsc.last_report(id); r ? r.status : 'unknown'}") {
131
+ wait_until(:fail_on_exceptions => true, :on_timeout => "Report generation timed out. Status: #{r = @nsc.last_report(id); r ? r.status : 'unknown'}") {
108
132
  if %w(Failed Aborted Unknown).include?(@nsc.last_report(id).status)
109
133
  raise "Report failed to generate! Status <#{@nsc.last_report(id).status}>"
110
134
  end
@@ -132,13 +156,13 @@ module NexposeServiceNow
132
156
  end
133
157
 
134
158
  def save_report(report_name, report_id, output_dir)
135
- @log.log_message 'Saving report...'
136
- local_file_name = self.class.get_filepath(report_name, output_dir)
137
- File.delete(local_file_name) if File.exists? local_file_name
159
+ @log.log_message 'Saving report...'
160
+ local_file_name = self.class.get_filepath(report_name, output_dir)
161
+ File.delete(local_file_name) if File.exists? local_file_name
138
162
 
139
- #log 'Downloading report...'
140
- report_details = @nsc.last_report(report_id)
141
- File.open(local_file_name, 'wb') do |f|
163
+ #log 'Downloading report...'
164
+ report_details = @nsc.last_report(report_id)
165
+ File.open(local_file_name, 'wb') do |f|
142
166
  f.write(@nsc.download(report_details.uri))
143
167
  end
144
168
 
@@ -146,35 +170,33 @@ module NexposeServiceNow
146
170
  # Refresh the connection
147
171
  @nsc = connect(@username, @password)
148
172
 
149
- #Got the report, cleanup server-side
150
- @nsc.delete_report_config(report_id)
173
+ #Got the report, cleanup server-side
174
+ @nsc.delete_report_config(report_id)
151
175
  rescue
152
- @log.log_error_message "Error deleting report"
176
+ @log.log_error_message 'Error deleting report'
153
177
  end
154
178
 
155
179
  local_file_name
156
180
  end
157
181
 
158
- def connect(username, password)
159
- begin
160
- console = Nexpose::Connection.new(@url, username, password)
161
- console.login
162
- @log.log_message 'Logged in.'
163
- rescue Exception => e
164
- @log.log_error_message 'Error logging in...'
165
- @log.log_error_message e
166
-
167
- $stderr.puts "ERROR: Could not log in. Check log and Nexpose settings.\n#{e}"
168
- exit -1
169
- end
170
-
171
- #@log.on_connect(@url, @port || 3780, console.session_id, "{}")
172
- console
173
- end
174
-
175
- def all_sites
176
- @nsc.sites.map { |s| s.id }
182
+ def connect(username, password)
183
+ begin
184
+ connection = Nexpose::Connection.new(@url, username, password, @port)
185
+ connection.login
186
+ @log.log_message 'Logged in.'
187
+ rescue Exception => e
188
+ msg = "ERROR: Could not log in. Check log and settings.\n#{e}"
189
+ @log.log_error_message msg
190
+ $stderr.puts msg
191
+ exit -1
192
+ end
193
+
194
+ connection
177
195
  end
178
196
 
197
+ # Pulls the collection IDs from Nexpose (e.g. asset groups, sites)
198
+ def collection_ids(collection_type)
199
+ @nsc.send("#{collection_type}s").map { |s| s.id }.sort
200
+ end
179
201
  end
180
202
  end
@@ -98,8 +98,8 @@ module NexposeServiceNow
98
98
  FROM dim_vulnerability_category dvc) dvc USING (vulnerability_id)
99
99
  WHERE date_modified >= '#{options[:vuln_query_date]}'"
100
100
  end
101
-
102
- #Filter by site.
101
+
102
+ # Filter by site.
103
103
  def self.assets(options={})
104
104
  "SELECT coalesce(host_name, CAST(dim_asset.asset_id as text)) as Name,
105
105
  dim_asset.ip_address,
@@ -116,7 +116,6 @@ module NexposeServiceNow
116
116
  fact_asset.pci_status
117
117
 
118
118
  FROM dim_asset
119
- JOIN (select * from dim_site_asset WHERE site_id=#{options['site_id']}) dsa USING (asset_id)
120
119
  JOIN fact_asset USING (asset_id)
121
120
  LEFT OUTER JOIN dim_operating_system on dim_asset.operating_system_id = dim_operating_system.operating_system_id
122
121
  LEFT OUTER JOIN dim_host_type USING (host_type_id)"
@@ -131,58 +130,133 @@ module NexposeServiceNow
131
130
  end
132
131
 
133
132
  def self.service_instance(options={})
134
- "SELECT ds.Service_Name, asset_id as Nexpose_ID, port, dp.protocol, dsf.name
133
+ 'SELECT ds.Service_Name, asset_id as Nexpose_ID, port, dp.protocol, dsf.name
135
134
  FROM fact_asset_scan_service
136
135
  LEFT OUTER JOIN (SELECT service_id, name as service_name FROM dim_service) ds USING (service_id)
137
136
  LEFT OUTER JOIN (SELECT service_fingerprint_id, name FROM dim_service_fingerprint) dsf USING (service_fingerprint_id)
138
137
  LEFT OUTER JOIN (SELECT protocol_id, name as protocol FROM dim_protocol) dp USING (protocol_id)
139
- WHERE scan_id = lastScan(asset_id)"
138
+ WHERE scan_id = lastScan(asset_id)'
140
139
  end
141
140
 
142
141
 
143
- #Need to wipe table each time
142
+ # Need to wipe table each time
144
143
  def self.group_accounts(options={})
145
- "SELECT asset_id as Nexpose_ID, daga.name as Group_Account_Name
144
+ 'SELECT asset_id as Nexpose_ID, daga.name as Group_Account_Name
146
145
  FROM dim_asset
147
- JOIN dim_asset_group_account daga USING (asset_id)"
146
+ JOIN dim_asset_group_account daga USING (asset_id)'
148
147
  end
149
148
 
150
- #Need to wipe table each time
149
+ # Need to wipe table each time
151
150
  def self.user_accounts(options={})
152
- "SELECT da.asset_id as Nexpose_ID, daua.name as User_Account_Name,
151
+ 'SELECT da.asset_id as Nexpose_ID, daua.name as User_Account_Name,
153
152
  daua.full_name as User_Account_Full_Name
154
153
 
155
154
  FROM dim_asset da
156
- JOIN dim_asset_user_account daua USING (asset_id)"
155
+ JOIN dim_asset_user_account daua USING (asset_id)'
157
156
  end
158
157
 
159
- #Need to wipe table each time
158
+ # Need to wipe table each time
160
159
  def self.asset_groups(options={})
161
- "SELECT asset_id as Nexpose_ID, dag.name as Asset_Group_Name,
160
+ 'SELECT asset_id as Nexpose_ID, dag.name as Asset_Group_Name,
162
161
  dag.dynamic_membership, dag.description
163
162
  FROM dim_asset_group_asset daga
164
- JOIN dim_asset_group dag on daga.asset_group_id = dag.asset_group_id"
163
+ JOIN dim_asset_group dag on daga.asset_group_id = dag.asset_group_id'
165
164
  end
166
165
 
167
- #Need to wipe table each time
166
+ # Need to wipe table each time
168
167
  def self.sites(options={})
169
- "SELECT asset_id as Nexpose_ID, ds.name as site_name
168
+ 'SELECT asset_id as Nexpose_ID, ds.name as site_name
170
169
  FROM dim_asset
171
170
  JOIN dim_site_asset dsa USING (asset_id)
172
171
  JOIN dim_site ds on dsa.site_id = ds.site_id
173
- ORDER BY ip_address"
172
+ ORDER BY ip_address'
174
173
  end
175
174
 
176
- #Need to wipe table each time
175
+ # Need to wipe table each time
177
176
  def self.tags(options={})
178
- "SELECT asset_id as Nexpose_ID, dt.tag_name
177
+ 'SELECT asset_id as Nexpose_ID, dt.tag_name
179
178
  FROM dim_tag_asset dta
180
- JOIN dim_tag dt on dta.tag_id = dt.tag_id"
179
+ JOIN dim_tag dt on dta.tag_id = dt.tag_id'
180
+ end
181
+
182
+ def self.generate_cve_filter(cves)
183
+ return '' if cves == nil || cves.empty?
184
+
185
+ cves = cves.map { |c| "reference='#{c}'" }.join(' OR ')
186
+
187
+ "JOIN (SELECT vulnerability_id, reference
188
+ FROM dim_vulnerability_reference
189
+ WHERE #{cves}) dvr USING (vulnerability_id)"
190
+ end
191
+
192
+ #TODO make sure that for max date first_discovered < date2
193
+ def self.generate_date_filter(dates, table_join=true)
194
+ return '' if dates == nil
195
+
196
+ # No filters applied, so no need to filter
197
+ return '' if dates.all? { |d| d == nil || d == '' }
198
+
199
+ min_date = dates.first
200
+ max_date = dates.last
201
+
202
+ # Version of vulnerable new items
203
+ unless table_join
204
+ filters = []
205
+ filters << "first_discovered >= '#{min_date}'" unless min_date.nil?
206
+ filters << "first_discovered <= '#{max_date}'" unless max_date.nil?
207
+ filters = filters.join(' AND ')
208
+ filters = "WHERE #{filters}"
209
+
210
+ return filters
211
+ end
212
+
213
+ date_filters = []
214
+
215
+ unless min_date.nil?
216
+ date_filters << "scan_finished > '#{min_date}'"
217
+ end
218
+ unless max_date.nil?
219
+ date_filters << "scan_started < '#{max_date}'"
220
+ end
221
+
222
+ condition = date_filters.join(' AND ')
223
+ "JOIN (SELECT scan_id, asset_id
224
+ FROM fact_asset_scan
225
+ WHERE #{condition}) fas
226
+ ON fas.asset_id = da.asset_ID AND
227
+ fas.scan_id = first_found"
228
+ end
229
+
230
+ def self.generate_cvss_filter(cvss_range)
231
+ return '' if cvss_range.nil? || cvss_range.last.nil?
232
+
233
+ cvss_min = cvss_range.first
234
+ cvss_max = cvss_range.last
235
+
236
+ # No need to join if not applying a filter
237
+ return '' if cvss_min.to_s == '0' && cvss_max.to_s == '10'
238
+
239
+ "JOIN (SELECT vulnerability_id, cvss_score
240
+ FROM dim_vulnerability
241
+ WHERE cvss_score >= #{cvss_min} AND cvss_score <= #{cvss_max}) dv
242
+ USING (vulnerability_id)"
181
243
  end
182
244
 
183
245
  def self.vulnerable_new_items(options={})
246
+ # self.send("vulnerable_new_items_per_#{options[:id_type]}", options)
247
+
248
+ standard_filter = if options[:id_type] == 'site'
249
+ "MIN(fasv.scan_id) > #{options[:delta]}"
250
+ else
251
+ "MIN(fasv.date) > '#{options[:delta]}'"
252
+ end
253
+
254
+ cve_filter = self.generate_cve_filter(options[:filters][:cve])
255
+ date_filter = self.generate_date_filter(options[:filters][:date], false)
256
+ cvss_filter = self.generate_cvss_filter(options[:filters][:cvss])
257
+
184
258
  "SELECT
185
- coalesce(subq.host_name, CAST(subq.asset_id as text)) Configuration_Item,
259
+ CAST(subq.asset_id as text) Configuration_Item,
186
260
  TRUE as Active,
187
261
  concat('R7_', subq.vulnerability_id) as Vulnerability,
188
262
  fasva.first_discovered as First_Found,
@@ -193,17 +267,25 @@ module NexposeServiceNow
193
267
  coalesce(NULLIF(favi.name,''), 'None') as Protocol
194
268
 
195
269
  FROM (
196
- SELECT fasv.asset_id, fasv.vulnerability_id, vulnerability_instances,
197
- MIN(fasv.scan_id) as first_found, MAX(fasv.scan_id) as latest_found,
270
+ SELECT fasv.asset_id, fasv.vulnerability_id, vulnerability_instances,
271
+ MIN(fasv.scan_id) as first_found, MAX(fasv.scan_id) as latest_found,
198
272
  s.current_scan, s.host_name, s.ip_address
199
273
  FROM fact_asset_scan_vulnerability_finding fasv
274
+
275
+ #{cve_filter}
276
+ #{cvss_filter}
277
+
200
278
  JOIN (
201
279
  SELECT asset_id, host_name, ip_address, lastScan(asset_id) AS current_scan FROM dim_asset
202
280
  ) s ON s.asset_id = fasv.asset_id
203
281
  GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan, s.host_name, s.ip_address, vulnerability_instances
204
- HAVING MIN(fasv.scan_id) > #{options[:last_scan_id]} AND MAX(fasv.scan_id)=current_scan
282
+ HAVING MAX(fasv.scan_id)=current_scan AND #{standard_filter}
205
283
  ) subq
206
- JOIN (select asset_id, vulnerability_id, first_discovered, most_recently_discovered from fact_asset_vulnerability_age) fasva ON fasva.asset_id = subq.asset_id AND fasva.vulnerability_id = subq.vulnerability_id
284
+
285
+ JOIN (select asset_id, vulnerability_id,
286
+ first_discovered, most_recently_discovered
287
+ from fact_asset_vulnerability_age
288
+ #{date_filter}) fasva USING (asset_id, vulnerability_id)
207
289
 
208
290
  JOIN (select DISTINCT on(asset_id, vulnerability_id) asset_id, scan_id, vulnerability_id, port, dp.name
209
291
  from fact_asset_vulnerability_instance
@@ -212,23 +294,52 @@ module NexposeServiceNow
212
294
  end
213
295
 
214
296
  def self.vulnerable_old_items(options={})
297
+ standard_filter = if options[:id_type] == 'site'
298
+ "MAX(fasv.scan_id) >= #{options[:delta]}"
299
+ else
300
+ "MAX(fasv.scan_id) >= scanAsOf(fasv.asset_id, '#{options[:delta]}') AND
301
+ lastScan(fasv.asset_id) > scanAsOf(fasv.asset_id, '#{options[:delta]}')"
302
+ end
303
+
304
+ cve_filter = self.generate_cve_filter(options[:filters][:cve])
305
+ date_filter = self.generate_date_filter(options[:filters][:date])
306
+ cvss_filter = self.generate_cvss_filter(options[:filters][:cvss])
307
+
308
+ # Only perform this operation is necessary
309
+ date_field = if date_filter.nil? || date_filter == ''
310
+ ''
311
+ else
312
+ 'MIN(fasv.scan_id) as first_found,'
313
+ end
314
+
215
315
  "SELECT
216
- coalesce(da.host_name, CAST(da.asset_id as text)) Configuration_Item,
316
+ CAST(da.asset_id as text) Configuration_Item,
217
317
  FALSE as Active,
218
318
  concat('R7_', subq.vulnerability_id) as Vulnerability,
219
319
  da.ip_address as IP_Address
220
320
  FROM (
221
- SELECT fasv.asset_id, fasv.vulnerability_id, MAX(fasv.scan_id) as latest_found,
321
+ SELECT fasv.asset_id, fasv.vulnerability_id,
322
+ #{date_field}
323
+ MAX(fasv.scan_id) as latest_found,
222
324
  s.current_scan, s.host_name, s.ip_address
223
325
  FROM fact_asset_scan_vulnerability_finding fasv
326
+
327
+ #{cve_filter}
328
+ #{cvss_filter}
329
+
224
330
  JOIN (
225
- SELECT asset_id, host_name, ip_address, lastScan(asset_id) AS current_scan FROM dim_asset
226
- ) s ON s.asset_id = fasv.asset_id
227
- GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan, s.host_name, s.ip_address, vulnerability_instances
228
- HAVING MAX(fasv.scan_id) < current_scan AND MAX(fasv.scan_id) >= #{options[:last_scan_id]}
229
- ) subq
331
+ SELECT asset_id, host_name, ip_address, lastScan(asset_id) AS current_scan FROM dim_asset
332
+ ) s ON s.asset_id = fasv.asset_id
333
+ GROUP BY fasv.asset_id, fasv.vulnerability_id, s.current_scan, s.host_name, s.ip_address, vulnerability_instances
334
+
335
+ HAVING MAX(fasv.scan_id) < current_scan
336
+ AND #{standard_filter}
337
+ ) subq
338
+
230
339
  JOIN dim_asset da ON subq.asset_id = da.asset_id
340
+ #{date_filter}
231
341
  ORDER BY da.ip_address"
342
+
232
343
  end
233
344
 
234
345
  def self.latest_scans(options={})
@@ -238,8 +349,8 @@ module NexposeServiceNow
238
349
  end
239
350
 
240
351
  def self.multiple_reports?(query_name)
241
- single_queries = ['vulnerabilities', 'vulnerability_category',
242
- 'vulnerability_references', 'latest_scans']
352
+ single_queries = %w(vulnerabilities vulnerability_category
353
+ vulnerability_references latest_scans)
243
354
  return !(single_queries.include? query_name.to_s)
244
355
  end
245
356
  end