abide_dev_utils 0.4.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,441 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'yaml'
4
+ require 'selenium-webdriver'
5
+ require 'abide_dev_utils/errors/comply'
6
+ require 'abide_dev_utils/gcloud'
7
+ require 'abide_dev_utils/output'
8
+ require 'abide_dev_utils/prompt'
9
+
10
+ module AbideDevUtils
11
+ # Holds module methods and a class for dealing with Puppet Comply
12
+ module Comply
13
+ include AbideDevUtils::Errors::Comply
14
+
15
+ def self.build_report(url, password, config = nil, **opts)
16
+ ReportScraper.new(url, config, **opts).build_report(password)
17
+ end
18
+
19
+ def self.check_for_regressions(url, password, config = nil, **opts)
20
+ current_report = build_report(url, password, config, **opts)
21
+ last_report = if opts.fetch(:remote_report_storage, '') == 'gcloud'
22
+ fetch_report
23
+ else
24
+ File.open(opts[:last_report], 'r', &:read)
25
+ end
26
+ result, details = good_comparison?(report_comparison(current_report, last_report))
27
+ if result
28
+ puts 'A-OK'
29
+ else
30
+ puts 'Uh-Oh'
31
+ puts details
32
+ end
33
+ end
34
+
35
+ # Class that uses Selenium WebDriver to gather scan reports from Puppet Comply
36
+ class ReportScraper
37
+ def initialize(url, config = nil, **opts)
38
+ @url = url
39
+ @config = config
40
+ @opts = opts
41
+ end
42
+
43
+ def timeout
44
+ @timeout ||= fetch_option(:timeout, 10).to_i
45
+ end
46
+
47
+ def username
48
+ @username ||= fetch_option(:username, 'comply')
49
+ end
50
+
51
+ def status
52
+ @status ||= fetch_option(:status)
53
+ end
54
+
55
+ def ignorelist
56
+ @ignorelist ||= fetch_option(:ignorelist, [])
57
+ end
58
+
59
+ def onlylist
60
+ @onlylist ||= fetch_option(:onlylist, [])
61
+ end
62
+
63
+ def screenshot_on_error
64
+ @screenshot_on_error ||= fetch_option(:screenshot_on_error, true)
65
+ end
66
+
67
+ def page_source_on_error
68
+ @page_source_on_error ||= fetch_option(:page_source_on_error, true)
69
+ end
70
+
71
+ def build_report(password)
72
+ connect(password)
73
+ scrape_report
74
+ ensure
75
+ driver.quit
76
+ end
77
+
78
+ def file_dir
79
+ @file_dir ||= File.expand_path('~/abide_dev_utils')
80
+ end
81
+
82
+ def file_dir=(path)
83
+ @file_dir = new_file_dir(path)
84
+ end
85
+
86
+ private
87
+
88
+ attr_reader :progress
89
+
90
+ def fetch_option(option, default = nil)
91
+ return @opts.fetch(option, default) if @config.nil?
92
+
93
+ @opts.key?(option) ? @opts[option] : @config.fetch(option, default)
94
+ end
95
+
96
+ def node_report_links
97
+ @node_report_links ||= find_node_report_links
98
+ end
99
+
100
+ def driver
101
+ @driver ||= new_driver
102
+ end
103
+
104
+ def output
105
+ AbideDevUtils::Output
106
+ end
107
+
108
+ def prompt
109
+ AbideDevUtils::Prompt
110
+ end
111
+
112
+ def new_progress(node_name)
113
+ @progress = AbideDevUtils::Output.progress title: "Building report for #{node_name}", total: nil
114
+ end
115
+
116
+ def new_driver
117
+ options = Selenium::WebDriver::Chrome::Options.new
118
+ options.args = @opts.fetch(:driveropts, %w[
119
+ --headless
120
+ --test-type
121
+ --disable-gpu
122
+ --no-first-run
123
+ --no-default-browser-check
124
+ --ignore-certificate-errors
125
+ --start-maximized
126
+ ])
127
+ output.simple 'Starting headless Chrome...'
128
+ Selenium::WebDriver.for(:chrome, options: options)
129
+ end
130
+
131
+ def find_element(subject = driver, **kwargs)
132
+ driver.manage.window.resize_to(1920, 1080)
133
+ subject.find_element(**kwargs)
134
+ end
135
+
136
+ def wait_on(ignore_nse: false, ignore: [Selenium::WebDriver::Error::NoSuchElementError], &block)
137
+ raise 'wait_on must be passed a block' unless block
138
+
139
+ value = nil
140
+ if ignore_nse
141
+ begin
142
+ Selenium::WebDriver::Wait.new(ignore: [], timeout: timeout).until do
143
+ value = yield
144
+ end
145
+ rescue Selenium::WebDriver::Error::NoSuchElementError
146
+ return value
147
+ rescue => e
148
+ raise_error(e)
149
+ end
150
+ else
151
+ begin
152
+ Selenium::WebDriver::Wait.new(ignore: ignore, timeout: timeout).until do
153
+ value = yield
154
+ end
155
+ rescue => e
156
+ raise_error(e)
157
+ end
158
+ end
159
+ value
160
+ end
161
+
162
+ def new_file_dir(path)
163
+ return File.expand_path(path) if Dir.exist?(File.expand_path(path))
164
+
165
+ create_dir = prompt.yes_no("Directory #{path} does not exist. Create directory?")
166
+ return unless create_dir
167
+
168
+ require 'fileutils'
169
+ FileUtils.mkdir_p path
170
+ end
171
+
172
+ def raise_error(err)
173
+ output.simple 'Something went wrong!'
174
+ if screenshot_on_error
175
+ output.simple 'Taking a screenshot of current page state...'
176
+ screenshot
177
+ end
178
+
179
+ if page_source_on_error
180
+ output.simple 'Saving page source of current page...'
181
+ page_source
182
+ end
183
+
184
+ driver.quit
185
+ raise err
186
+ end
187
+
188
+ def screenshot
189
+ driver.save_screenshot(File.join(file_dir, "comply_error_#{Time.now.to_i}.png"))
190
+ rescue Errno::ENOENT
191
+ save_default = prompt.yes_no(
192
+ "Directory #{file_dir} does not exist. Save screenshot to current directory?"
193
+ )
194
+ driver.save_screenshot(File.join(File.expand_path('.'), "comply_error_#{Time.now.to_i}.png")) if save_default
195
+ end
196
+
197
+ def page_source
198
+ File.open(File.join(file_dir, "comply_error_#{Time.now.to_i}.txt"), 'w') { |f| f.write(driver.page_source) }
199
+ rescue Errno::ENOENT
200
+ save_default = prompt.yes_no(
201
+ "Directory #{file_dir} does not exist. Save page source to current directory?"
202
+ )
203
+ if save_default
204
+ File.open(File.join(File.expand_path('.'), "comply_error_#{Time.now.to_i}.html"), 'w') do |f|
205
+ f.write(driver.page_source)
206
+ end
207
+ end
208
+ end
209
+
210
+ def bypass_ssl_warning_page
211
+ wait_on(ignore_nse: true) do
212
+ find_element(id: 'details-button').click
213
+ find_element(id: 'proceed-link').click
214
+ end
215
+ end
216
+
217
+ def login_to_comply(password)
218
+ output.simple "Logging into Comply at #{@url}..."
219
+ wait_on { driver.find_element(id: 'username') }
220
+ find_element(id: 'username').send_keys username
221
+ find_element(id: 'password').send_keys password
222
+ find_element(id: 'kc-login').click
223
+ error_text = wait_on(ignore_nse: true) { find_element(class: 'kc-feedback-text').text }
224
+ return if error_text.nil? || error_text.empty?
225
+
226
+ raise ComplyLoginFailedError, error_text
227
+ end
228
+
229
+ def find_node_report_links
230
+ output.simple 'Finding nodes with scan reports...'
231
+ hosts = wait_on { find_element(class: 'metric-containers-failed-hosts-count') }
232
+ table = find_element(hosts, class: 'rc-table')
233
+ table_body = find_element(table, tag_name: 'tbody')
234
+ wait_on { table_body.find_elements(tag_name: 'a') }
235
+ end
236
+
237
+ def connect(password)
238
+ output.simple "Connecting to #{@url}..."
239
+ driver.get(@url)
240
+ bypass_ssl_warning_page
241
+ login_to_comply(password)
242
+ end
243
+
244
+ def normalize_cis_rec_name(name)
245
+ nstr = name.downcase
246
+ nstr.delete!('(/|\\|\+|:|\'|")')
247
+ nstr.gsub!(/(\s|\(|\)|-|\.)/, '_')
248
+ nstr.strip!
249
+ nstr
250
+ end
251
+
252
+ def scrape_report
253
+ output.simple 'Building scan reports, this may take a while...'
254
+ all_checks = {}
255
+ original_window = driver.window_handle
256
+ if !onlylist.empty?
257
+ node_report_links.reject! { |l| !onlylist.include?(l.text) }
258
+ elsif !ignorelist.empty?
259
+ node_report_links.reject! { |l| ignorelist.include?(l.text) }
260
+ end
261
+ node_report_links.each do |link|
262
+ begin
263
+ node_name = link.text
264
+ new_progress(node_name)
265
+ link_url = link.attribute('href')
266
+ driver.manage.new_window(:tab)
267
+ progress.increment
268
+ wait_on { driver.window_handles.length == 2 }
269
+ progress.increment
270
+ driver.switch_to.window driver.window_handles[1]
271
+ driver.get(link_url)
272
+ wait_on { find_element(class: 'details-scan-info') }
273
+ progress.increment
274
+ wait_on { find_element(class: 'details-table') }
275
+ progress.increment
276
+ report = { 'scan_results' => {} }
277
+ scan_info_table = find_element(class: 'details-scan-info')
278
+ scan_info_table_rows = scan_info_table.find_elements(tag_name: 'tr')
279
+ progress.increment
280
+ check_table_body = find_element(tag_name: 'tbody')
281
+ check_table_rows = check_table_body.find_elements(tag_name: 'tr')
282
+ progress.increment
283
+ scan_info_table_rows.each do |row|
284
+ key = find_element(row, tag_name: 'h5').text
285
+ value = find_element(row, tag_name: 'strong').text
286
+ report[key.downcase.tr(':', '').tr(' ', '_')] = value
287
+ progress.increment
288
+ end
289
+ check_table_rows.each do |row|
290
+ chk_objs = row.find_elements(tag_name: 'td')
291
+ chk_objs.map!(&:text)
292
+ if status.nil? || status.include?(chk_objs[1].downcase)
293
+ name_parts = chk_objs[0].match(/^([0-9.]+) (.+)$/)
294
+ key = normalize_cis_rec_name(name_parts[2])
295
+ unless report['scan_results'].key?(chk_objs[1])
296
+ report['scan_results'][chk_objs[1]] = {}
297
+ end
298
+ report['scan_results'][chk_objs[1]][key] = {
299
+ 'name' => name_parts[2].chomp,
300
+ 'number' => name_parts[1].chomp
301
+ }
302
+ end
303
+ progress.increment
304
+ end
305
+ all_checks[node_name] = report
306
+ driver.close
307
+ output.simple "Created report for #{node_name}"
308
+ rescue => e
309
+ raise_error(e)
310
+ ensure
311
+ driver.switch_to.window original_window
312
+ end
313
+ end
314
+ all_checks
315
+ end
316
+ end
317
+
318
+ # Contains multiple NodeScanReport objects
319
+ class ScanReport
320
+ def from_yaml(report)
321
+ @scan_report = if report.is_a? Hash
322
+ report
323
+ elsif File.file?(report)
324
+ File.open(report.to_s, 'r') { |f| YAML.safe_load(f.read) }
325
+ else
326
+ YAML.safe_load(report)
327
+ end
328
+ build_node_scan_reports
329
+ end
330
+
331
+ private
332
+
333
+ def build_node_scan_reports
334
+ node_scan_reports = []
335
+ @scan_report.each do |node_name, node_hash|
336
+ node_scan_reports << NodeScanReport.new(node_name, node_hash)
337
+ end
338
+ node_scan_reports.sort_by(&:name)
339
+ end
340
+ end
341
+
342
+ # Class representation of a Comply node scan report
343
+ class NodeScanReport
344
+ attr_reader :name, :passing, :failing, :not_checked, :informational, :benchmark, :last_scan, :profile
345
+
346
+ DIFF_PROPERTIES = %i[passing failing not_checked informational].freeze
347
+
348
+ def initialize(node_name, node_hash)
349
+ @name = node_name
350
+ @hash = node_hash
351
+ @passing = node_hash.dig('scan_results', 'Pass') || {}
352
+ @failing = node_hash.dig('scan_results', 'Fail') || {}
353
+ @not_checked = node_hash.dig('scan_results', 'Not checked') || {}
354
+ @informational = node_hash.dig('scan_results', 'Informational') || {}
355
+ @benchmark = node_hash['benchmark']
356
+ @last_scan = node_hash['last_scan']
357
+ @profile = node_hash.fetch('custom_profile', nil) || node_hash.fetch('profile', nil)
358
+ end
359
+
360
+ def diff(other)
361
+ diff = {}
362
+ DIFF_PROPERTIES.each do |prop|
363
+ diff[prop] = send("#{prop.to_s}_equal?".to_sym, other.send(prop)) ? {} : property_diff(prop, other)
364
+ end
365
+ diff
366
+ end
367
+
368
+ def method_missing(method_name, *args, &_block)
369
+ case method_name
370
+ when method_name.match?(/^(passing|failing|not_checked|informational)_equal?$/)
371
+ property_equal?(method_name.delete_suffix('_equal?'), *args)
372
+ when method_name.match?(/^(to_h|to_yaml)$/)
373
+ @hash.send(method_name.to_sym)
374
+ end
375
+ end
376
+
377
+ def respond_to_missing?(method_name, _include_private = false)
378
+ method_name.match?(/^(((passing|failing|not_checked|informational)_equal?)|to_h|to_yaml)$/)
379
+ end
380
+
381
+ private
382
+
383
+ def property_diff(property, other)
384
+ {
385
+ self: send(property).keys - other.send(property).keys,
386
+ other: other.send(property).keys - send(property).keys
387
+ }
388
+ end
389
+
390
+ def property_equal?(property, other_property)
391
+ send(property.to_sym) == other_property
392
+ end
393
+ end
394
+
395
+ def self.storage_bucket
396
+ @storage_bucket ||= AbideDevUtils::GCloud.storage_bucket
397
+ end
398
+
399
+ def self.fetch_report
400
+ report = storage_bucket.file('comply_report.yaml')
401
+ report.download.read
402
+ end
403
+
404
+ def self.upload_report(report)
405
+ file_to_upload = report.is_a?(Hash) ? report.to_yaml : report
406
+ storage_bucket.create_file(file_to_upload, 'comply_report.yaml')
407
+ end
408
+
409
+ def self.report_comparison(current, last)
410
+ current_report = ScanReport.new.from_yaml(current)
411
+ last_report = ScanReport.new.from_yaml(last)
412
+
413
+ comparison = []
414
+ current_report.zip(last_report).each do |cr, lr|
415
+ comparison << { cr.name => { diff: {}, node_presense: :new } } if lr.nil?
416
+ comparison << { lr.name => { diff: {}, node_presense: :dropped } } if cr.nil?
417
+ comparison << { cr.name => { diff: cr.diff(lr), node_presence: :same } } unless cr.nil? || lr.nil?
418
+ end
419
+ comparison.inject(&:merge)
420
+ end
421
+
422
+ def self.good_comparison?(report_comparison)
423
+ good = true
424
+ not_good = {}
425
+ report_comparison.each do |node_name, report|
426
+ next if report[:diff].empty?
427
+
428
+ not_good[node_name] = {}
429
+ unless report[:diff][:passing][:other].empty?
430
+ good = false
431
+ not_good[node_name][:new_not_passing] = report[:diff][:passing][:other]
432
+ end
433
+ unless report[:diff][:failing][:self].empty?
434
+ good = false
435
+ not_good[node_name][:new_failing] = report[:diff][:failing][:self]
436
+ end
437
+ end
438
+ [good, not_good]
439
+ end
440
+ end
441
+ end
@@ -7,18 +7,41 @@ module AbideDevUtils
7
7
  DEFAULT_PATH = "#{File.expand_path('~')}/.abide_dev.yaml"
8
8
 
9
9
  def self.to_h(path = DEFAULT_PATH)
10
+ return {} unless File.file?(path)
11
+
12
+ h = YAML.safe_load(File.open(path), [Symbol])
13
+ h.transform_keys(&:to_sym)
14
+ end
15
+
16
+ def to_h(path = DEFAULT_PATH)
17
+ return {} unless File.file?(path)
18
+
10
19
  h = YAML.safe_load(File.open(path), [Symbol])
11
20
  h.transform_keys(&:to_sym)
12
21
  end
13
22
 
14
23
  def self.config_section(section, path = DEFAULT_PATH)
15
24
  h = to_h(path)
16
- s = h[section.to_sym]
25
+ s = h.fetch(section.to_sym, nil)
26
+ return {} if s.nil?
27
+
28
+ s.transform_keys(&:to_sym)
29
+ end
30
+
31
+ def config_section(section, path = DEFAULT_PATH)
32
+ h = to_h(path)
33
+ s = h.fetch(section.to_sym, nil)
34
+ return {} if s.nil?
35
+
17
36
  s.transform_keys(&:to_sym)
18
37
  end
19
38
 
20
39
  def self.fetch(key, default = nil, path = DEFAULT_PATH)
21
40
  to_h(path).fetch(key, default)
22
41
  end
42
+
43
+ def fetch(key, default = nil, path = DEFAULT_PATH)
44
+ to_h(path).fetch(key, default)
45
+ end
23
46
  end
24
47
  end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'abide_dev_utils/errors/base'
4
+
5
+ module AbideDevUtils
6
+ module Errors
7
+ module Comply
8
+ class ComplyLoginFailedError < GenericError
9
+ @default = 'Failed to login to Comply:'
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'abide_dev_utils/errors/base'
4
+
5
+ module AbideDevUtils
6
+ module Errors
7
+ module GCloud
8
+ class MissingCredentialsError < GenericError
9
+ @default = <<~EOERR
10
+ Storage credentials not given. Please set environment variable ABIDE_GCLOUD_CREDENTIALS.
11
+ EOERR
12
+ end
13
+
14
+ class MissingProjectError < GenericError
15
+ @default = <<~EOERR
16
+ Storage project not given. Please set the environment variable ABIDE_GCLOUD_PROJECT.
17
+ EOERR
18
+ end
19
+
20
+ class MissingBucketNameError < GenericError
21
+ @default = <<~EOERR
22
+ Storage bucket name not given. Please set the environment variable ABIDE_GCLOUD_BUCKET.
23
+ EOERR
24
+ end
25
+ end
26
+ end
27
+ end
@@ -24,6 +24,18 @@ module AbideDevUtils
24
24
  class FailedToCreateFileError < GenericError
25
25
  @default = 'Failed to create file:'
26
26
  end
27
+
28
+ class ClassFileNotFoundError < GenericError
29
+ @default = 'Class file was not found:'
30
+ end
31
+
32
+ class ClassDeclarationNotFoundError < GenericError
33
+ @default = 'Class declaration was not found:'
34
+ end
35
+
36
+ class InvalidClassNameError < GenericError
37
+ @default = 'Not a valid Puppet class name:'
38
+ end
27
39
  end
28
40
  end
29
41
  end
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'abide_dev_utils/errors/base'
4
+ require 'abide_dev_utils/errors/comply'
5
+ require 'abide_dev_utils/errors/gcloud'
4
6
  require 'abide_dev_utils/errors/general'
5
7
  require 'abide_dev_utils/errors/jira'
6
8
  require 'abide_dev_utils/errors/xccdf'
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'abide_dev_utils/errors/gcloud'
4
+
5
+ module AbideDevUtils
6
+ module GCloud
7
+ include AbideDevUtils::Errors::GCloud
8
+
9
+ def self.storage_bucket(name: nil, project: nil, credentials: nil)
10
+ raise MissingProjectError if project.nil? && ENV['ABIDE_GCLOUD_PROJECT'].nil?
11
+ raise MissingCredentialsError if credentials.nil? && ENV['ABIDE_GCLOUD_CREDENTIALS'].nil?
12
+ raise MissingBucketNameError if name.nil? && ENV['ABIDE_GCLOUD_BUCKET'].nil?
13
+
14
+ require 'google/cloud/storage'
15
+ @bucket = Google::Cloud::Storage.new(
16
+ project_id: project || ENV['ABIDE_GCLOUD_PROJECT'],
17
+ credentials: credentials || ENV['ABIDE_GCLOUD_CREDENTIALS']
18
+ ).bucket(name || ENV['ABIDE_GCLOUD_BUCKET'])
19
+ end
20
+ end
21
+ end
@@ -86,6 +86,8 @@ module AbideDevUtils
86
86
 
87
87
  def self.client(options: {})
88
88
  opts = merge_options(options)
89
+ return client_from_prompts if opts.empty?
90
+
89
91
  opts[:username] = AbideDevUtils::Prompt.username if opts[:username].nil?
90
92
  opts[:password] = AbideDevUtils::Prompt.password if opts[:password].nil?
91
93
  opts[:site] = AbideDevUtils::Prompt.single_line('Jira URL') if opts[:site].nil?
@@ -133,6 +135,16 @@ module AbideDevUtils
133
135
  end
134
136
  end
135
137
 
138
+ # def self.new_issues_from_comply_report(client, project, report, dry_run: false)
139
+ # dr_prefix = dry_run ? 'DRY RUN: ' : ''
140
+ # i_attrs = all_project_issues_attrs(project)
141
+ # rep_sums = summaries_from_coverage_report(report)
142
+ # rep_sums.each do |k, v|
143
+ # next if summary_exist?(k, i_attrs)
144
+
145
+ # progress = AbideDevUtils::Output.progress(title: "#{dr_prefix}Creating Tasks", total: nil)
146
+ # v.each do |s|
147
+
136
148
  def self.merge_options(options)
137
149
  config.merge(options)
138
150
  end
@@ -165,6 +177,11 @@ module AbideDevUtils
165
177
  summaries.transform_keys { |k| "#{COV_PARENT_SUMMARY_PREFIX}#{benchmark}-#{k}"}
166
178
  end
167
179
 
180
+ # def self.summaries_from_comply_report(report)
181
+ # summaries = {}
182
+ # report.each do |_, v|
183
+ # end
184
+
168
185
  class Dummy
169
186
  def attrs
170
187
  { 'fields' => {
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module AbideDevUtils
4
+ module Mixins
5
+ # mixin methods for the Hash data type
6
+ module Hash
7
+ def deep_copy
8
+ Marshal.load(Marshal.dump(self))
9
+ end
10
+
11
+ def diff(other)
12
+ dup.delete_if { |k, v| other[k] == v }.merge!(other.dup.delete_if { |k, _| key?(k) })
13
+ end
14
+ end
15
+ end
16
+ end