nexpose_servicenow 0.4.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,234 @@
1
+ require_relative './nexpose_helper'
2
+ require_relative './nx_logger'
3
+
4
+ module NexposeServiceNow
5
+ class HistoricalData
6
+ REPORT_FILE_NAME = "Nexpose-ServiceNow-latest_scans.csv"
7
+ STORED_FILE_NAME = "last_scan_data.csv"
8
+ TIMESTAMP_FILE_NAME = "last_vuln_run.csv"
9
+
10
+ def initialize(options)
11
+ @local_dir = File.expand_path(options[:output_dir])
12
+ @ids = options[:nexpose_ids]
13
+
14
+ @local_file = File.join(@local_dir, STORED_FILE_NAME)
15
+ @remote_file = File.join(@local_dir, REPORT_FILE_NAME)
16
+ @timestamp_file = File.join(@local_dir, TIMESTAMP_FILE_NAME)
17
+
18
+ @log = NexposeServiceNow::NxLogger.instance
19
+ @log.log_message "Retrieving environment variables."
20
+ end
21
+
22
+ #Filters the saved report down to the sites being queried
23
+ #This can then be used as a basis to update last_scan_data
24
+ def filter_report
25
+ #Create a full last_scan_data if it doesn't already exist
26
+ create_last_scan_data unless File.exist? @local_file
27
+
28
+ @log.log_message 'Filtering report down sites which will be queried'
29
+
30
+ remote_csv = load_scan_id_report
31
+ site_ids = @ids.map(&:to_s)
32
+ filtered_csv = remote_csv.delete_if { |r| !site_ids.include?(r['site_id']) }
33
+ File.open(@remote_file, 'w') do |f|
34
+ f.write(remote_csv.to_csv)
35
+ end
36
+
37
+ puts filtered_csv
38
+ end
39
+
40
+ #Reads the downloaded report containing LATEST scan IDs
41
+ def load_scan_id_report
42
+ @log.log_message "Loading scan data report"
43
+ unless File.exists? @remote_file
44
+ @log.log_message "No existing report file found."
45
+ return nil
46
+ end
47
+ CSV.read(@remote_file, headers: true)
48
+ end
49
+
50
+ #Loads the last scan data file as CSV.
51
+ #It may be necessary to create one first.
52
+ def load_last_scan_data
53
+ @log.log_message "Loading last scan data"
54
+
55
+ create_last_scan_data unless File.exist? @local_file
56
+ CSV.read(@local_file, headers: true)
57
+ end
58
+
59
+ def last_scan_ids(sites)
60
+ return [] if !File.exist? @local_file
61
+
62
+ csv = load_last_scan_data
63
+ last_scan_ids = {}
64
+ sites.each do |id|
65
+ row = csv.find { |r| r['site_id'] == id.to_s } || { 'last_scan_id' => '0' }
66
+ last_scan_ids[id.to_s] = row['last_scan_id']
67
+ end
68
+
69
+ last_scan_ids
70
+ end
71
+
72
+ #Compares stored scan IDs versus remote scan IDs.
73
+ #This determines which scans are included as filters.
74
+ def sites_to_scan
75
+ return @ids unless File.exist? @remote_file
76
+
77
+ @log.log_message 'Filtering for sites with new scans'
78
+
79
+ remote_csv = CSV.read(@remote_file, headers: true)
80
+ local_csv = load_last_scan_data
81
+
82
+ filtered_sites = []
83
+
84
+ @ids.each do |id|
85
+ remote_scan_id = remote_csv.find { |r| r['site_id'] == id.to_s } || {}
86
+ remote_scan_id = remote_scan_id['last_scan_id'] || 1
87
+
88
+ local_scan_id = local_csv.find { |r| r['site_id'] == id.to_s }
89
+ local_scan_id = local_scan_id['last_scan_id']
90
+
91
+ filtered_sites << id if local_scan_id.to_i < remote_scan_id.to_i
92
+ end
93
+
94
+ @ids = filtered_sites
95
+ end
96
+
97
+ #Creates a base last scan data file from a downloaded report
98
+ def create_last_scan_data
99
+ @log.log_message 'Creating base last scan data file'
100
+
101
+ csv = load_scan_id_report
102
+ csv.delete('finished')
103
+ csv.each { |l| l['last_scan_id'] = 0 }
104
+
105
+ save_last_scan_data(csv)
106
+ end
107
+
108
+ #Updates only the rows that were affected by this scan
109
+ def update_last_scan_data
110
+ @log.log_message "Updating last scan data"
111
+
112
+ if !(File.exist? @local_file) && !(File.exist? @remote_file)
113
+ @log.log_message 'Last scan data does not exist yet.'
114
+ return
115
+ end
116
+
117
+ updated_csv = load_last_scan_data
118
+ remote_csv = load_scan_id_report
119
+
120
+ #merge changes in from remote_csv
121
+ remote_csv.each do |row|
122
+ updated_row = updated_csv.find { |r| r['site_id'] == row['site_id'] }
123
+ updated_row['last_scan_id'] = row['last_scan_id']
124
+ end
125
+
126
+ save_last_scan_data(updated_csv)
127
+
128
+ #puts updated_csv
129
+ end
130
+
131
+ #Overwrite the last scan data file with new csv
132
+ def save_last_scan_data(csv)
133
+ @log.log_message 'Saving last scan data'
134
+ File.open(@local_file, 'w') do |f|
135
+ f.write(csv.to_csv)
136
+ end
137
+ end
138
+
139
+ def create_last_vuln_data(time=nil, sites=[])
140
+ @log.log_message 'Creating last vulnerability scan time file.'
141
+
142
+ time ||= Time.new(1985)
143
+ time = time.strftime("%Y-%m-%d") if time.class.to_s == 'Time'
144
+
145
+ file = File.expand_path(@timestamp_file)
146
+
147
+ CSV.open(file, 'w') do |csv|
148
+ csv << ['Last Scan Time', 'Sites']
149
+ csv << [time, sites.join(',')]
150
+ end
151
+ end
152
+
153
+ #Current IDs are inserted into the updated CSV file.
154
+ def last_vuln_run
155
+ @log.log_message 'Retrieving the last vulnerability timestamp'
156
+
157
+ create_last_vuln_data if !File.exist? @timestamp_file
158
+
159
+ file = File.expand_path(@timestamp_file)
160
+ csv = CSV.open(file, headers:true)
161
+ last_run = csv.readline['Last Scan Time']
162
+ csv.close
163
+
164
+ last_run
165
+ end
166
+
167
+ #Experimental
168
+ #These should probably return strings that can be mlog'd
169
+ def log_and_print(message)
170
+ puts message
171
+ @log.log_message message unless @log.nil?
172
+ end
173
+
174
+ def log_and_error(message)
175
+ $stderr.puts "ERROR: #{message}"
176
+ @log.log_message unless @log.nil?
177
+ end
178
+
179
+ def set_last_scan(nexpose_id, scan_id)
180
+ unless File.exist? @local_file
181
+ log_and_error 'Last scan data does not exist.'
182
+ log_and_error 'Can\'t set last scan data without existing file.'
183
+ exit -1
184
+ end
185
+
186
+ csv = load_last_scan_data
187
+ row = csv.find { |r| r['site_id'] == nexpose_id }
188
+
189
+ if row.nil?
190
+ csv << [nexpose_id, scan_id]
191
+ else
192
+ row['last_scan_id'] = scan_id
193
+ end
194
+
195
+ save_last_scan_data csv
196
+
197
+ log_and_print 'Last scan data updated.'
198
+ end
199
+
200
+ def set_last_vuln(date, sites=nil)
201
+ create_last_vuln_data(date, sites)
202
+ log_and_print 'Last vuln data updated.'
203
+ end
204
+
205
+ def remove_local_file(filename)
206
+ unless File.exist? filename
207
+ log_and_error 'Can\'t remove file.'
208
+ log_and_error "File #{filename} cannot be located."
209
+ exit -1
210
+ end
211
+
212
+ new_name = "#{filename}.#{Time.new.strftime('%Y-%m-%d.%H:%M:%S')}"
213
+ begin
214
+ #Delete existing file with same name
215
+ File.delete new_name if File.exist? new_name
216
+ File.rename(filename, new_name)
217
+ rescue Exception => e
218
+ log_and_error "Error removing file:\n#{e}"
219
+ exit -1
220
+ end
221
+
222
+ log_and_print "File #{filename} removed"
223
+ end
224
+
225
+ def remove_last_scan_data
226
+ remove_local_file @local_file
227
+ end
228
+
229
+ def remove_last_vuln_data
230
+ remove_local_file @timestamp_file
231
+ end
232
+
233
+ end
234
+ end
@@ -0,0 +1,162 @@
1
+ require 'nexpose'
2
+ require 'fileutils'
3
+ require_relative './queries'
4
+ require_relative './nx_logger'
5
+
6
+ module NexposeServiceNow
7
+ class NexposeHelper
8
+ def initialize(url, port, username, password)
9
+ @log = NexposeServiceNow::NxLogger.instance
10
+ @url = url
11
+ @port = port
12
+ @nsc = connect(username, password)
13
+
14
+ @timeout = 7200
15
+ end
16
+
17
+ def self.get_report_names(query_name, ids)
18
+ unless Queries.multiple_reports?(query_name)
19
+ return [ id: -1, report_name: get_report_name(query_name) ]
20
+ end
21
+
22
+ ids.map { |id| { id: id, report_name: get_report_name(query_name, id) }}
23
+ end
24
+
25
+ def self.get_report_name(query_name, id=nil)
26
+ name = "Nexpose-ServiceNow-#{query_name}"
27
+ name += "-#{id}" if Queries.multiple_reports?(query_name) && !id.nil?
28
+ name
29
+ end
30
+
31
+ def self.get_filepath(report_name, output_dir)
32
+ path = File.join output_dir, "#{report_name}.csv"
33
+ File.expand_path path
34
+ end
35
+
36
+ def create_report(query_name, ids, id_type, output_dir, query_options={})
37
+ output_dir = File.expand_path(output_dir.to_s)
38
+
39
+ #A single report doesn't use site filters
40
+ ids = [-1] unless Queries.multiple_reports?(query_name)
41
+
42
+ reports = []
43
+ ids.each do |id|
44
+ report_name = self.class.get_report_name(query_name, id)
45
+ clean_up_reports(report_name)
46
+
47
+ delta_options = create_query_options(query_options, id)
48
+
49
+ query = Queries.send(query_name, delta_options)
50
+ report_id = generate_config(query, report_name, [id], id_type)
51
+ run_report(report_id)
52
+ reports << save_report(report_name, report_id, output_dir)
53
+ end
54
+
55
+ return reports
56
+ end
57
+
58
+ def create_query_options(query_options, nexpose_id=nil)
59
+ options = {}
60
+ options[:vuln_query_date] = query_options[:vuln_query_date]
61
+
62
+ return options if nexpose_id == nil || nexpose_id == -1
63
+ return 0 if query_options[:last_scans].empty?
64
+ options[:last_scan_id] = "#{query_options[:last_scans][nexpose_id] || 0}"
65
+
66
+ @log.log_message("Query options: #{options}")
67
+
68
+ options
69
+ end
70
+
71
+ def clean_up_reports(report_name)
72
+ #log 'Deleting existing report...'
73
+ reports = @nsc.list_reports
74
+ reports.select! { |r| r.name.start_with? report_name }
75
+ reports.each do |r|
76
+ @nsc.delete_report_config(r.config_id)
77
+ end
78
+ end
79
+
80
+ def generate_config(query, report_name, ids, id_type)
81
+ @log.log_message "Generating query config with name #{report_name}..."
82
+ report_config = Nexpose::ReportConfig.new(report_name, nil, 'sql')
83
+ report_config.add_filter('version', '2.0.1')
84
+ report_config.add_filter('query', query)
85
+
86
+ ids.each { |id| report_config.add_filter(id_type, id) if id != -1 }
87
+ report_id = report_config.save(@nsc, false)
88
+ end
89
+
90
+ def run_report(report_id)
91
+ #log 'Running report...'
92
+ @nsc.generate_report(report_id, false)
93
+ wait_for_report(report_id)
94
+ end
95
+
96
+ def wait_for_report(id)
97
+ wait_until(:fail_on_exceptions => TRUE, :on_timeout => "Report generation timed out. Status: #{r = @nsc.last_report(id); r ? r.status : 'unknown'}") {
98
+ if %w(Failed Aborted Unknown).include?(@nsc.last_report(id).status)
99
+ raise "Report failed to generate! Status <#{@nsc.last_report(id).status}>"
100
+ end
101
+ @nsc.last_report(id).status == 'Generated'
102
+ }
103
+ end
104
+
105
+ def wait_until(options = {})
106
+ polling_interval = 15
107
+ time_limit = Time.now + @timeout
108
+ loop do
109
+ begin
110
+ val = yield
111
+ return val if val
112
+ rescue Exception => error
113
+ raise error if options[:fail_on_exceptions]
114
+ end
115
+ if Time.now >= time_limit
116
+ raise options[:on_timeout] if options[:on_timeout]
117
+ error ||= 'Timed out waiting for condition.'
118
+ raise error
119
+ end
120
+ sleep polling_interval
121
+ end
122
+ end
123
+
124
+ def save_report(report_name, report_id, output_dir)
125
+ @log.log_message 'Saving report...'
126
+ local_file_name = self.class.get_filepath(report_name, output_dir)
127
+ File.delete(local_file_name) if File.exists? local_file_name
128
+
129
+ #log 'Downloading report...'
130
+ report_details = @nsc.last_report(report_id)
131
+ File.open(local_file_name, 'wb') do |f|
132
+ f.write(@nsc.download(report_details.uri))
133
+ end
134
+
135
+ #Got the report, cleanup server-side
136
+ @nsc.delete_report_config(report_id)
137
+ local_file_name
138
+ end
139
+
140
+ def connect(username, password)
141
+ begin
142
+ console = Nexpose::Connection.new(@url, username, password)
143
+ console.login
144
+ @log.log_message 'Logged in.'
145
+ rescue Exception => e
146
+ @log.log_error_message 'Error logging in...'
147
+ @log.log_error_message e
148
+
149
+ $stderr.puts "ERROR: Could not log in. Check log and Nexpose settings.\n#{e}"
150
+ exit -1
151
+ end
152
+
153
+ #@log.on_connect(@url, @port || 3780, console.session_id, "{}")
154
+ console
155
+ end
156
+
157
+ def all_sites
158
+ @nsc.sites.map { |s| s.id }
159
+ end
160
+
161
+ end
162
+ end
@@ -0,0 +1,166 @@
1
+ require 'fileutils'
2
+ require 'json'
3
+ require 'net/http'
4
+ require 'singleton'
5
+
6
+ module NexposeServiceNow
7
+ class NxLogger
8
+ include Singleton
9
+ LOG_PATH = "./logs/rapid7_%s.log"
10
+ KEY_FORMAT = "external.integration.%s"
11
+ PRODUCT_FORMAT = "%s_%s"
12
+
13
+ DEFAULT_LOG = 'integration'
14
+ PRODUCT_RANGE = 4..30
15
+ KEY_RANGE = 3..15
16
+
17
+ ENDPOINT = '/data/external/statistic/'
18
+
19
+ def initialize()
20
+ create_calls
21
+ @logger_file = get_log_path @product
22
+ setup_logging(true, 'info')
23
+ end
24
+
25
+ def setup_statistics_collection(vendor, product_name, gem_version)
26
+ begin
27
+ @statistic_key = get_statistic_key vendor
28
+ @product = get_product product_name, gem_version
29
+ rescue => e
30
+ #Continue
31
+ end
32
+ end
33
+
34
+ def setup_logging(enabled, log_level = 'info', stdout=false)
35
+ @stdout = stdout
36
+
37
+ log_message('Logging disabled.') unless enabled || @log.nil?
38
+ @enabled = enabled
39
+ return unless @enabled
40
+
41
+ @logger_file = get_log_path @product
42
+
43
+ require 'logger'
44
+ directory = File.dirname(@logger_file)
45
+ FileUtils.mkdir_p(directory) unless File.directory?(directory)
46
+ io = IO.for_fd(IO.sysopen(@logger_file, 'a'), 'a')
47
+ io.autoclose = false
48
+ io.sync = true
49
+ @log = Logger.new(io, 'weekly')
50
+ @log.level = if log_level.to_s.casecmp('info') == 0
51
+ Logger::INFO
52
+ else
53
+ Logger::DEBUG
54
+ end
55
+ log_message("Logging enabled at level <#{log_level}>")
56
+ end
57
+
58
+ def create_calls
59
+ levels = [:info, :debug, :error, :warn]
60
+ levels.each do |level|
61
+ method_name =
62
+ define_singleton_method("log_#{level.to_s}_message") do |message|
63
+ puts message if @stdout
64
+ @log.send(level, message) unless !@enabled || @log.nil?
65
+ end
66
+ end
67
+ end
68
+
69
+ def log_message(message)
70
+ log_info_message message
71
+ end
72
+
73
+ def log_stat_message(message)
74
+ end
75
+
76
+ def get_log_path(product)
77
+ product.downcase! unless product.nil?
78
+ File.join(File.dirname(__FILE__), LOG_PATH % (product || DEFAULT_LOG))
79
+ end
80
+
81
+ def get_statistic_key(vendor)
82
+ if vendor.nil? || vendor.length < KEY_RANGE.min
83
+ log_stat_message("Vendor length is below minimum of <#{KEY_RANGE}>")
84
+ return nil
85
+ end
86
+
87
+ vendor.gsub!('-', '_')
88
+ vendor.slice! vendor.rindex('_') until vendor.count('_') <= 1
89
+
90
+ vendor.delete! "^A-Za-z0-9\_"
91
+
92
+ KEY_FORMAT % vendor[0...KEY_RANGE.max].downcase
93
+ end
94
+
95
+ def get_product(product, version)
96
+ return nil if ((product.nil? || product.empty?) ||
97
+ (version.nil? || version.empty?))
98
+
99
+ product.gsub!('-', '_')
100
+ product.slice! product.rindex('_') until product.count('_') <= 1
101
+
102
+ product.delete! "^A-Za-z0-9\_"
103
+ version.delete! "^A-Za-z0-9\.\-"
104
+
105
+ product = (PRODUCT_FORMAT % [product, version])[0...PRODUCT_RANGE.max]
106
+
107
+ product.slice! product.rindex(/[A-Z0-9]/i)+1..-1
108
+
109
+ if product.length < PRODUCT_RANGE.min
110
+ log_stat_message("Product length below minimum <#{PRODUCT_RANGE.min}>.")
111
+ return nil
112
+ end
113
+ product.downcase
114
+ end
115
+
116
+ def generate_payload(statistic_value='')
117
+ product_name, separator, version = @product.to_s.rpartition('_')
118
+ payload_value = {'version' => version}.to_json
119
+
120
+ payload = {'statistic-key' => @statistic_key.to_s,
121
+ 'statistic-value' => payload_value,
122
+ 'product' => product_name}
123
+ JSON.generate(payload)
124
+ end
125
+
126
+ def send(nexpose_address, nexpose_port, session_id, payload)
127
+ header = {'Content-Type' => 'application/json',
128
+ 'nexposeCCSessionID' => session_id,
129
+ 'Cookie' => "nexposeCCSessionID=#{session_id}"}
130
+ req = Net::HTTP::Put.new(ENDPOINT, header)
131
+ req.body = payload
132
+ http_instance = Net::HTTP.new(nexpose_address, nexpose_port)
133
+ http_instance.use_ssl = true
134
+ http_instance.verify_mode = OpenSSL::SSL::VERIFY_NONE
135
+ response = http_instance.start { |http| http.request(req) }
136
+ log_stat_message "Received code #{response.code} from Nexpose console."
137
+ log_stat_message "Received message #{response.msg} from Nexpose console."
138
+ log_stat_message 'Finished sending statistics data to Nexpose.'
139
+
140
+ response.code
141
+ end
142
+
143
+ def on_connect(nexpose_address, nexpose_port, session_id, value)
144
+ log_stat_message 'Sending statistics data to Nexpose'
145
+
146
+ if @product.nil? || @statistic_key.nil?
147
+ log_stat_message('Invalid product name and/or statistics key.')
148
+ log_stat_message('Statistics collection not enabled.')
149
+ return
150
+ end
151
+
152
+ begin
153
+ payload = generate_payload value
154
+ send(nexpose_address, nexpose_port, session_id, payload)
155
+ rescue => e
156
+ #Let the program continue
157
+ end
158
+ end
159
+
160
+ #Used by net library for debugging
161
+ def <<(value)
162
+ log_debug_message(value)
163
+ end
164
+
165
+ end
166
+ end