abide_dev_utils 0.4.1 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,441 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'yaml'
4
+ require 'selenium-webdriver'
5
+ require 'abide_dev_utils/errors/comply'
6
+ require 'abide_dev_utils/gcloud'
7
+ require 'abide_dev_utils/output'
8
+ require 'abide_dev_utils/prompt'
9
+
10
+ module AbideDevUtils
11
+ # Holds module methods and a class for dealing with Puppet Comply
12
+ module Comply
13
+ include AbideDevUtils::Errors::Comply
14
+
15
+ def self.build_report(url, password, config = nil, **opts)
16
+ ReportScraper.new(url, config, **opts).build_report(password)
17
+ end
18
+
19
+ def self.check_for_regressions(url, password, config = nil, **opts)
20
+ current_report = build_report(url, password, config, **opts)
21
+ last_report = if opts.fetch(:remote_report_storage, '') == 'gcloud'
22
+ fetch_report
23
+ else
24
+ File.open(opts[:last_report], 'r', &:read)
25
+ end
26
+ result, details = good_comparison?(report_comparison(current_report, last_report))
27
+ if result
28
+ puts 'A-OK'
29
+ else
30
+ puts 'Uh-Oh'
31
+ puts details
32
+ end
33
+ end
34
+
35
+ # Class that uses Selenium WebDriver to gather scan reports from Puppet Comply
36
+ class ReportScraper
37
+ def initialize(url, config = nil, **opts)
38
+ @url = url
39
+ @config = config
40
+ @opts = opts
41
+ end
42
+
43
+ def timeout
44
+ @timeout ||= fetch_option(:timeout, 10).to_i
45
+ end
46
+
47
+ def username
48
+ @username ||= fetch_option(:username, 'comply')
49
+ end
50
+
51
+ def status
52
+ @status ||= fetch_option(:status)
53
+ end
54
+
55
+ def ignorelist
56
+ @ignorelist ||= fetch_option(:ignorelist, [])
57
+ end
58
+
59
+ def onlylist
60
+ @onlylist ||= fetch_option(:onlylist, [])
61
+ end
62
+
63
+ def screenshot_on_error
64
+ @screenshot_on_error ||= fetch_option(:screenshot_on_error, true)
65
+ end
66
+
67
+ def page_source_on_error
68
+ @page_source_on_error ||= fetch_option(:page_source_on_error, true)
69
+ end
70
+
71
+ def build_report(password)
72
+ connect(password)
73
+ scrape_report
74
+ ensure
75
+ driver.quit
76
+ end
77
+
78
+ def file_dir
79
+ @file_dir ||= File.expand_path('~/abide_dev_utils')
80
+ end
81
+
82
+ def file_dir=(path)
83
+ @file_dir = new_file_dir(path)
84
+ end
85
+
86
+ private
87
+
88
+ attr_reader :progress
89
+
90
+ def fetch_option(option, default = nil)
91
+ return @opts.fetch(option, default) if @config.nil?
92
+
93
+ @opts.key?(option) ? @opts[option] : @config.fetch(option, default)
94
+ end
95
+
96
+ def node_report_links
97
+ @node_report_links ||= find_node_report_links
98
+ end
99
+
100
+ def driver
101
+ @driver ||= new_driver
102
+ end
103
+
104
+ def output
105
+ AbideDevUtils::Output
106
+ end
107
+
108
+ def prompt
109
+ AbideDevUtils::Prompt
110
+ end
111
+
112
+ def new_progress(node_name)
113
+ @progress = AbideDevUtils::Output.progress title: "Building report for #{node_name}", total: nil
114
+ end
115
+
116
+ def new_driver
117
+ options = Selenium::WebDriver::Chrome::Options.new
118
+ options.args = @opts.fetch(:driveropts, %w[
119
+ --headless
120
+ --test-type
121
+ --disable-gpu
122
+ --no-first-run
123
+ --no-default-browser-check
124
+ --ignore-certificate-errors
125
+ --start-maximized
126
+ ])
127
+ output.simple 'Starting headless Chrome...'
128
+ Selenium::WebDriver.for(:chrome, options: options)
129
+ end
130
+
131
+ def find_element(subject = driver, **kwargs)
132
+ driver.manage.window.resize_to(1920, 1080)
133
+ subject.find_element(**kwargs)
134
+ end
135
+
136
+ def wait_on(ignore_nse: false, ignore: [Selenium::WebDriver::Error::NoSuchElementError], &block)
137
+ raise 'wait_on must be passed a block' unless block
138
+
139
+ value = nil
140
+ if ignore_nse
141
+ begin
142
+ Selenium::WebDriver::Wait.new(ignore: [], timeout: timeout).until do
143
+ value = yield
144
+ end
145
+ rescue Selenium::WebDriver::Error::NoSuchElementError
146
+ return value
147
+ rescue => e
148
+ raise_error(e)
149
+ end
150
+ else
151
+ begin
152
+ Selenium::WebDriver::Wait.new(ignore: ignore, timeout: timeout).until do
153
+ value = yield
154
+ end
155
+ rescue => e
156
+ raise_error(e)
157
+ end
158
+ end
159
+ value
160
+ end
161
+
162
+ def new_file_dir(path)
163
+ return File.expand_path(path) if Dir.exist?(File.expand_path(path))
164
+
165
+ create_dir = prompt.yes_no("Directory #{path} does not exist. Create directory?")
166
+ return unless create_dir
167
+
168
+ require 'fileutils'
169
+ FileUtils.mkdir_p path
170
+ end
171
+
172
+ def raise_error(err)
173
+ output.simple 'Something went wrong!'
174
+ if screenshot_on_error
175
+ output.simple 'Taking a screenshot of current page state...'
176
+ screenshot
177
+ end
178
+
179
+ if page_source_on_error
180
+ output.simple 'Saving page source of current page...'
181
+ page_source
182
+ end
183
+
184
+ driver.quit
185
+ raise err
186
+ end
187
+
188
+ def screenshot
189
+ driver.save_screenshot(File.join(file_dir, "comply_error_#{Time.now.to_i}.png"))
190
+ rescue Errno::ENOENT
191
+ save_default = prompt.yes_no(
192
+ "Directory #{file_dir} does not exist. Save screenshot to current directory?"
193
+ )
194
+ driver.save_screenshot(File.join(File.expand_path('.'), "comply_error_#{Time.now.to_i}.png")) if save_default
195
+ end
196
+
197
+ def page_source
198
+ File.open(File.join(file_dir, "comply_error_#{Time.now.to_i}.txt"), 'w') { |f| f.write(driver.page_source) }
199
+ rescue Errno::ENOENT
200
+ save_default = prompt.yes_no(
201
+ "Directory #{file_dir} does not exist. Save page source to current directory?"
202
+ )
203
+ if save_default
204
+ File.open(File.join(File.expand_path('.'), "comply_error_#{Time.now.to_i}.html"), 'w') do |f|
205
+ f.write(driver.page_source)
206
+ end
207
+ end
208
+ end
209
+
210
+ def bypass_ssl_warning_page
211
+ wait_on(ignore_nse: true) do
212
+ find_element(id: 'details-button').click
213
+ find_element(id: 'proceed-link').click
214
+ end
215
+ end
216
+
217
+ def login_to_comply(password)
218
+ output.simple "Logging into Comply at #{@url}..."
219
+ wait_on { driver.find_element(id: 'username') }
220
+ find_element(id: 'username').send_keys username
221
+ find_element(id: 'password').send_keys password
222
+ find_element(id: 'kc-login').click
223
+ error_text = wait_on(ignore_nse: true) { find_element(class: 'kc-feedback-text').text }
224
+ return if error_text.nil? || error_text.empty?
225
+
226
+ raise ComplyLoginFailedError, error_text
227
+ end
228
+
229
+ def find_node_report_links
230
+ output.simple 'Finding nodes with scan reports...'
231
+ hosts = wait_on { find_element(class: 'metric-containers-failed-hosts-count') }
232
+ table = find_element(hosts, class: 'rc-table')
233
+ table_body = find_element(table, tag_name: 'tbody')
234
+ wait_on { table_body.find_elements(tag_name: 'a') }
235
+ end
236
+
237
+ def connect(password)
238
+ output.simple "Connecting to #{@url}..."
239
+ driver.get(@url)
240
+ bypass_ssl_warning_page
241
+ login_to_comply(password)
242
+ end
243
+
244
+ def normalize_cis_rec_name(name)
245
+ nstr = name.downcase
246
+ nstr.delete!('(/|\\|\+|:|\'|")')
247
+ nstr.gsub!(/(\s|\(|\)|-|\.)/, '_')
248
+ nstr.strip!
249
+ nstr
250
+ end
251
+
252
+ def scrape_report
253
+ output.simple 'Building scan reports, this may take a while...'
254
+ all_checks = {}
255
+ original_window = driver.window_handle
256
+ if !onlylist.empty?
257
+ node_report_links.reject! { |l| !onlylist.include?(l.text) }
258
+ elsif !ignorelist.empty?
259
+ node_report_links.reject! { |l| ignorelist.include?(l.text) }
260
+ end
261
+ node_report_links.each do |link|
262
+ begin
263
+ node_name = link.text
264
+ new_progress(node_name)
265
+ link_url = link.attribute('href')
266
+ driver.manage.new_window(:tab)
267
+ progress.increment
268
+ wait_on { driver.window_handles.length == 2 }
269
+ progress.increment
270
+ driver.switch_to.window driver.window_handles[1]
271
+ driver.get(link_url)
272
+ wait_on { find_element(class: 'details-scan-info') }
273
+ progress.increment
274
+ wait_on { find_element(class: 'details-table') }
275
+ progress.increment
276
+ report = { 'scan_results' => {} }
277
+ scan_info_table = find_element(class: 'details-scan-info')
278
+ scan_info_table_rows = scan_info_table.find_elements(tag_name: 'tr')
279
+ progress.increment
280
+ check_table_body = find_element(tag_name: 'tbody')
281
+ check_table_rows = check_table_body.find_elements(tag_name: 'tr')
282
+ progress.increment
283
+ scan_info_table_rows.each do |row|
284
+ key = find_element(row, tag_name: 'h5').text
285
+ value = find_element(row, tag_name: 'strong').text
286
+ report[key.downcase.tr(':', '').tr(' ', '_')] = value
287
+ progress.increment
288
+ end
289
+ check_table_rows.each do |row|
290
+ chk_objs = row.find_elements(tag_name: 'td')
291
+ chk_objs.map!(&:text)
292
+ if status.nil? || status.include?(chk_objs[1].downcase)
293
+ name_parts = chk_objs[0].match(/^([0-9.]+) (.+)$/)
294
+ key = normalize_cis_rec_name(name_parts[2])
295
+ unless report['scan_results'].key?(chk_objs[1])
296
+ report['scan_results'][chk_objs[1]] = {}
297
+ end
298
+ report['scan_results'][chk_objs[1]][key] = {
299
+ 'name' => name_parts[2].chomp,
300
+ 'number' => name_parts[1].chomp
301
+ }
302
+ end
303
+ progress.increment
304
+ end
305
+ all_checks[node_name] = report
306
+ driver.close
307
+ output.simple "Created report for #{node_name}"
308
+ rescue => e
309
+ raise_error(e)
310
+ ensure
311
+ driver.switch_to.window original_window
312
+ end
313
+ end
314
+ all_checks
315
+ end
316
+ end
317
+
318
+ # Contains multiple NodeScanReport objects
319
+ class ScanReport
320
+ def from_yaml(report)
321
+ @scan_report = if report.is_a? Hash
322
+ report
323
+ elsif File.file?(report)
324
+ File.open(report.to_s, 'r') { |f| YAML.safe_load(f.read) }
325
+ else
326
+ YAML.safe_load(report)
327
+ end
328
+ build_node_scan_reports
329
+ end
330
+
331
+ private
332
+
333
+ def build_node_scan_reports
334
+ node_scan_reports = []
335
+ @scan_report.each do |node_name, node_hash|
336
+ node_scan_reports << NodeScanReport.new(node_name, node_hash)
337
+ end
338
+ node_scan_reports.sort_by(&:name)
339
+ end
340
+ end
341
+
342
+ # Class representation of a Comply node scan report
343
+ class NodeScanReport
344
+ attr_reader :name, :passing, :failing, :not_checked, :informational, :benchmark, :last_scan, :profile
345
+
346
+ DIFF_PROPERTIES = %i[passing failing not_checked informational].freeze
347
+
348
+ def initialize(node_name, node_hash)
349
+ @name = node_name
350
+ @hash = node_hash
351
+ @passing = node_hash.dig('scan_results', 'Pass') || {}
352
+ @failing = node_hash.dig('scan_results', 'Fail') || {}
353
+ @not_checked = node_hash.dig('scan_results', 'Not checked') || {}
354
+ @informational = node_hash.dig('scan_results', 'Informational') || {}
355
+ @benchmark = node_hash['benchmark']
356
+ @last_scan = node_hash['last_scan']
357
+ @profile = node_hash.fetch('custom_profile', nil) || node_hash.fetch('profile', nil)
358
+ end
359
+
360
+ def diff(other)
361
+ diff = {}
362
+ DIFF_PROPERTIES.each do |prop|
363
+ diff[prop] = send("#{prop.to_s}_equal?".to_sym, other.send(prop)) ? {} : property_diff(prop, other)
364
+ end
365
+ diff
366
+ end
367
+
368
+ def method_missing(method_name, *args, &_block)
369
+ case method_name
370
+ when method_name.match?(/^(passing|failing|not_checked|informational)_equal?$/)
371
+ property_equal?(method_name.delete_suffix('_equal?'), *args)
372
+ when method_name.match?(/^(to_h|to_yaml)$/)
373
+ @hash.send(method_name.to_sym)
374
+ end
375
+ end
376
+
377
+ def respond_to_missing?(method_name, _include_private = false)
378
+ method_name.match?(/^(((passing|failing|not_checked|informational)_equal?)|to_h|to_yaml)$/)
379
+ end
380
+
381
+ private
382
+
383
+ def property_diff(property, other)
384
+ {
385
+ self: send(property).keys - other.send(property).keys,
386
+ other: other.send(property).keys - send(property).keys
387
+ }
388
+ end
389
+
390
+ def property_equal?(property, other_property)
391
+ send(property.to_sym) == other_property
392
+ end
393
+ end
394
+
395
+ def self.storage_bucket
396
+ @storage_bucket ||= AbideDevUtils::GCloud.storage_bucket
397
+ end
398
+
399
+ def self.fetch_report
400
+ report = storage_bucket.file('comply_report.yaml')
401
+ report.download.read
402
+ end
403
+
404
+ def self.upload_report(report)
405
+ file_to_upload = report.is_a?(Hash) ? report.to_yaml : report
406
+ storage_bucket.create_file(file_to_upload, 'comply_report.yaml')
407
+ end
408
+
409
+ def self.report_comparison(current, last)
410
+ current_report = ScanReport.new.from_yaml(current)
411
+ last_report = ScanReport.new.from_yaml(last)
412
+
413
+ comparison = []
414
+ current_report.zip(last_report).each do |cr, lr|
415
+ comparison << { cr.name => { diff: {}, node_presense: :new } } if lr.nil?
416
+ comparison << { lr.name => { diff: {}, node_presense: :dropped } } if cr.nil?
417
+ comparison << { cr.name => { diff: cr.diff(lr), node_presence: :same } } unless cr.nil? || lr.nil?
418
+ end
419
+ comparison.inject(&:merge)
420
+ end
421
+
422
+ def self.good_comparison?(report_comparison)
423
+ good = true
424
+ not_good = {}
425
+ report_comparison.each do |node_name, report|
426
+ next if report[:diff].empty?
427
+
428
+ not_good[node_name] = {}
429
+ unless report[:diff][:passing][:other].empty?
430
+ good = false
431
+ not_good[node_name][:new_not_passing] = report[:diff][:passing][:other]
432
+ end
433
+ unless report[:diff][:failing][:self].empty?
434
+ good = false
435
+ not_good[node_name][:new_failing] = report[:diff][:failing][:self]
436
+ end
437
+ end
438
+ [good, not_good]
439
+ end
440
+ end
441
+ end
@@ -7,18 +7,41 @@ module AbideDevUtils
7
7
  DEFAULT_PATH = "#{File.expand_path('~')}/.abide_dev.yaml"
8
8
 
9
9
  def self.to_h(path = DEFAULT_PATH)
10
+ return {} unless File.file?(path)
11
+
12
+ h = YAML.safe_load(File.open(path), [Symbol])
13
+ h.transform_keys(&:to_sym)
14
+ end
15
+
16
+ def to_h(path = DEFAULT_PATH)
17
+ return {} unless File.file?(path)
18
+
10
19
  h = YAML.safe_load(File.open(path), [Symbol])
11
20
  h.transform_keys(&:to_sym)
12
21
  end
13
22
 
14
23
  def self.config_section(section, path = DEFAULT_PATH)
15
24
  h = to_h(path)
16
- s = h[section.to_sym]
25
+ s = h.fetch(section.to_sym, nil)
26
+ return {} if s.nil?
27
+
28
+ s.transform_keys(&:to_sym)
29
+ end
30
+
31
+ def config_section(section, path = DEFAULT_PATH)
32
+ h = to_h(path)
33
+ s = h.fetch(section.to_sym, nil)
34
+ return {} if s.nil?
35
+
17
36
  s.transform_keys(&:to_sym)
18
37
  end
19
38
 
20
39
  def self.fetch(key, default = nil, path = DEFAULT_PATH)
21
40
  to_h(path).fetch(key, default)
22
41
  end
42
+
43
+ def fetch(key, default = nil, path = DEFAULT_PATH)
44
+ to_h(path).fetch(key, default)
45
+ end
23
46
  end
24
47
  end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'abide_dev_utils/errors/base'
4
+
5
+ module AbideDevUtils
6
+ module Errors
7
+ module Comply
8
+ class ComplyLoginFailedError < GenericError
9
+ @default = 'Failed to login to Comply:'
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'abide_dev_utils/errors/base'
4
+
5
+ module AbideDevUtils
6
+ module Errors
7
+ module GCloud
8
+ class MissingCredentialsError < GenericError
9
+ @default = <<~EOERR
10
+ Storage credentials not given. Please set environment variable ABIDE_GCLOUD_CREDENTIALS.
11
+ EOERR
12
+ end
13
+
14
+ class MissingProjectError < GenericError
15
+ @default = <<~EOERR
16
+ Storage project not given. Please set the environment variable ABIDE_GCLOUD_PROJECT.
17
+ EOERR
18
+ end
19
+
20
+ class MissingBucketNameError < GenericError
21
+ @default = <<~EOERR
22
+ Storage bucket name not given. Please set the environment variable ABIDE_GCLOUD_BUCKET.
23
+ EOERR
24
+ end
25
+ end
26
+ end
27
+ end
@@ -24,6 +24,18 @@ module AbideDevUtils
24
24
  class FailedToCreateFileError < GenericError
25
25
  @default = 'Failed to create file:'
26
26
  end
27
+
28
+ class ClassFileNotFoundError < GenericError
29
+ @default = 'Class file was not found:'
30
+ end
31
+
32
+ class ClassDeclarationNotFoundError < GenericError
33
+ @default = 'Class declaration was not found:'
34
+ end
35
+
36
+ class InvalidClassNameError < GenericError
37
+ @default = 'Not a valid Puppet class name:'
38
+ end
27
39
  end
28
40
  end
29
41
  end
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'abide_dev_utils/errors/base'
4
+ require 'abide_dev_utils/errors/comply'
5
+ require 'abide_dev_utils/errors/gcloud'
4
6
  require 'abide_dev_utils/errors/general'
5
7
  require 'abide_dev_utils/errors/jira'
6
8
  require 'abide_dev_utils/errors/xccdf'
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'abide_dev_utils/errors/gcloud'
4
+
5
+ module AbideDevUtils
6
+ module GCloud
7
+ include AbideDevUtils::Errors::GCloud
8
+
9
+ def self.storage_bucket(name: nil, project: nil, credentials: nil)
10
+ raise MissingProjectError if project.nil? && ENV['ABIDE_GCLOUD_PROJECT'].nil?
11
+ raise MissingCredentialsError if credentials.nil? && ENV['ABIDE_GCLOUD_CREDENTIALS'].nil?
12
+ raise MissingBucketNameError if name.nil? && ENV['ABIDE_GCLOUD_BUCKET'].nil?
13
+
14
+ require 'google/cloud/storage'
15
+ @bucket = Google::Cloud::Storage.new(
16
+ project_id: project || ENV['ABIDE_GCLOUD_PROJECT'],
17
+ credentials: credentials || ENV['ABIDE_GCLOUD_CREDENTIALS']
18
+ ).bucket(name || ENV['ABIDE_GCLOUD_BUCKET'])
19
+ end
20
+ end
21
+ end
@@ -86,6 +86,8 @@ module AbideDevUtils
86
86
 
87
87
  def self.client(options: {})
88
88
  opts = merge_options(options)
89
+ return client_from_prompts if opts.empty?
90
+
89
91
  opts[:username] = AbideDevUtils::Prompt.username if opts[:username].nil?
90
92
  opts[:password] = AbideDevUtils::Prompt.password if opts[:password].nil?
91
93
  opts[:site] = AbideDevUtils::Prompt.single_line('Jira URL') if opts[:site].nil?
@@ -133,6 +135,16 @@ module AbideDevUtils
133
135
  end
134
136
  end
135
137
 
138
+ # def self.new_issues_from_comply_report(client, project, report, dry_run: false)
139
+ # dr_prefix = dry_run ? 'DRY RUN: ' : ''
140
+ # i_attrs = all_project_issues_attrs(project)
141
+ # rep_sums = summaries_from_coverage_report(report)
142
+ # rep_sums.each do |k, v|
143
+ # next if summary_exist?(k, i_attrs)
144
+
145
+ # progress = AbideDevUtils::Output.progress(title: "#{dr_prefix}Creating Tasks", total: nil)
146
+ # v.each do |s|
147
+
136
148
  def self.merge_options(options)
137
149
  config.merge(options)
138
150
  end
@@ -165,6 +177,11 @@ module AbideDevUtils
165
177
  summaries.transform_keys { |k| "#{COV_PARENT_SUMMARY_PREFIX}#{benchmark}-#{k}"}
166
178
  end
167
179
 
180
+ # def self.summaries_from_comply_report(report)
181
+ # summaries = {}
182
+ # report.each do |_, v|
183
+ # end
184
+
168
185
  class Dummy
169
186
  def attrs
170
187
  { 'fields' => {
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module AbideDevUtils
4
+ module Mixins
5
+ # mixin methods for the Hash data type
6
+ module Hash
7
+ def deep_copy
8
+ Marshal.load(Marshal.dump(self))
9
+ end
10
+
11
+ def diff(other)
12
+ dup.delete_if { |k, v| other[k] == v }.merge!(other.dup.delete_if { |k, _| key?(k) })
13
+ end
14
+ end
15
+ end
16
+ end