abide_dev_utils 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71c8c65ce60fc385fdc289f00b98328f35e0b343b397864fea49010683ab274c
-  data.tar.gz: 1060f161729e0330efadf391bc35f9a84544522cc443364833c56beeec8fb308
+  metadata.gz: a27976b9f740b67261fa080eeba927bc6fc98e73ffb592fafdd91d4d612cc51f
+  data.tar.gz: 64778a0d476e2f96d70a14ab318b517c597aa3c5e33afd8939173385b013fe6c
 SHA512:
-  metadata.gz: 608138ff9fecf9835094848f5d7967b8b9ba87d9da766e943e1ec053936cc605e823befa0e5c893579dd4a8ad9a3b8dff49b4129624e9a338cb5cf0775f32272
-  data.tar.gz: 3610c0fae0872a7883a5608e703c0a025ff529b39e025315f984c40f6a63bc09206eaa9e874a11a180c2ad392c9afb14cd2eb18974129fecfee8eed4bbb2470e
+  metadata.gz: e31df40e6dd34a57156ff517d39fcc036f5655ce2f31bf86b1ebb66754304575b7379e0a3751ae378bc81bc1520dbe095255012dc7e10b24e4f6c3f4a85e6551
+  data.tar.gz: c9dd204d55a37d389c0c8a4d3d281310b87241cbbd149ca029a3fa6a38fd8414bcb350467cd348cdd330fe946e6ee75625bfe2828319ad25d9eb8ed5cc37a9a3

data/README.md

@@ -29,6 +29,10 @@ Issues and pull requests are welcome!
 
 * Create Jira issues in bulk from coverage reports
 
+### Puppet Comply Report Generation
+
+* Allows you ot programatically generate compliance reports from Puppet Comply
+
 ### Supports Configuration via Local YAML file
 
 * Fully configurable via the `~/.abide_dev.yaml` file

data/abide_dev_utils.gemspec

@@ -37,6 +37,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'puppet', '>= 6.19'
   spec.add_dependency 'jira-ruby', '~> 2.1'
   spec.add_dependency 'ruby-progressbar', '~> 1.11'
+  spec.add_dependency 'selenium-webdriver', '~> 4.0.0.beta4'
 
   # Dev dependencies
   spec.add_development_dependency 'bundler'

data/lib/abide_dev_utils/cli.rb

@@ -3,6 +3,7 @@
 require 'cmdparse'
 require 'abide_dev_utils/version'
 require 'abide_dev_utils/constants'
+require 'abide_dev_utils/cli/comply'
 require 'abide_dev_utils/cli/puppet'
 require 'abide_dev_utils/cli/xccdf'
 require 'abide_dev_utils/cli/test'
@@ -21,6 +22,7 @@ module Abide
       parser.main_options.banner = ROOT_CMD_BANNER
       parser.add_command(CmdParse::HelpCommand.new, default: true)
       parser.add_command(CmdParse::VersionCommand.new(add_switches: true))
+      parser.add_command(ComplyCommand.new)
       parser.add_command(PuppetCommand.new)
       parser.add_command(XccdfCommand.new)
       parser.add_command(TestCommand.new)

data/lib/abide_dev_utils/cli/abstract.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'abide_dev_utils/config'
+
 module Abide
   module CLI
     # @abstract

data/lib/abide_dev_utils/cli/comply.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'abide_dev_utils/comply'
+require 'abide_dev_utils/cli/abstract'
+
+module Abide
+  module CLI
+    class ComplyCommand < AbideCommand
+      CMD_NAME = 'comply'
+      CMD_SHORT = 'Commands related to Puppet Comply'
+      CMD_LONG = 'Namespace for commands related to Puppet Comply'
+      def initialize
+        super(CMD_NAME, CMD_SHORT, CMD_LONG, takes_commands: true)
+        add_command(ComplyReportCommand.new)
+      end
+    end
+
+    class ComplyReportCommand < AbideCommand
+      CMD_NAME = 'report'
+      CMD_SHORT = 'Generates a yaml report of Puppet Comply scan results'
+      CMD_LONG = <<~LONGCMD
+        Generates a yaml file that shows the scan results of all nodes in Puppet Comply.
+        This command utilizes Selenium WebDriver and the Google Chrome browser to automate
+        clicking through the Comply UI and building a report. In order to use this command,
+        you MUST have Google Chrome installed and you MUST install the chromedriver binary.
+        More info and instructions can be found here:
+        https://www.selenium.dev/documentation/en/getting_started_with_webdriver/.
+      LONGCMD
+      CMD_COMPLY_URL = 'The URL (including https://) of Puppet Comply'
+      CMD_COMPLY_PASSWORD = 'The password for Puppet Comply'
+      OPT_STATUS_DESC = <<~EODESC
+        A comma-separated list of check statuses to ONLY include in the report.
+        Valid statuses are: pass, fail, error, notapplicable, notchecked, unknown, informational
+      EODESC
+      OPT_IGNORE_NODES = <<~EOIGN
+        A comma-separated list of node certnames to ignore building reports for. This
+        options is mutually exclusive with --only and, if both are set, --only will take precedence
+        over this option.
+      EOIGN
+      OPT_ONLY_NODES = <<~EOONLY
+        A comma-separated list of node certnames to ONLY build reports for. No other
+        nodes will have reports built for them except the ones specified. This option
+        is mutually exclusive with --ignore and, if both are set, this options will
+        take precedence over --ignore.
+      EOONLY
+      def initialize
+        super(CMD_NAME, CMD_SHORT, CMD_LONG, takes_commands: false)
+        argument_desc(COMPLY_URL: CMD_COMPLY_URL, COMPLY_PASSWORD: CMD_COMPLY_PASSWORD)
+        options.on('-o [FILE]', '--out-file [FILE]', 'Path to save the report') { |f| @data[:file] = f }
+        options.on('-u [USERNAME]', '--username [USERNAME]', 'The username for Comply (defaults to comply)') do |u|
+          @data[:username] = u
+        end
+        options.on('-s [STATUS]', '--status [STATUS]', OPT_STATUS_DESC) do |s|
+          status_array = s.nil? ? nil : s.split(',').map(&:downcase)
+          status_array&.map! { |i| i == 'notchecked' ? 'not checked' : i }
+          @data[:status] = status_array
+        end
+        options.on('-O [CERTNAME]', '--only [CERTNAME]', OPT_ONLY_NODES) do |o|
+          only_array = o.nil? ? nil : s.split(',').map(&:downcase)
+          @data[:only] = only_array
+        end
+        options.on('-I [CERTNAME]', '--ignore [CERTNAME]', OPT_IGNORE_NODES) do |i|
+          ignore_array = i.nil? ? nil : i.split(',').map(&:downcase)
+          @data[:ignore] = ignore_array
+        end
+      end
+
+      def help_arguments
+        <<~ARGHELP
+          Arguments:
+              COMPLY_URL        #{CMD_COMPLY_URL}
+              COMPLY_PASSWORD   #{CMD_COMPLY_PASSWORD}
+
+        ARGHELP
+      end
+
+      def execute(comply_url = nil, comply_password = nil)
+        Abide::CLI::VALIDATE.filesystem_path(`command -v chromedriver`.strip)
+        conf = config_section('comply')
+        comply_url = conf.fetch(:url) if comply_url.nil?
+        comply_password = comply_password.nil? ? conf.fetch(:password, Abide::CLI::PROMPT.password) : comply_password
+        username = @data.fetch(:username, nil).nil? ? conf.fetch(:username, 'comply') : @data[:username]
+        status = @data.fetch(:status, nil).nil? ? conf.fecth(:status, nil) : @data[:status]
+        ignorelist = @data.fetch(:ignore, nil).nil? ? conf.fetch(:ignore, nil) : @data[:ignore]
+        onlylist = @data.fetch(:only, nil).nil? ? conf.fetch(:only, nil) : @data[:only]
+        report = AbideDevUtils::Comply.scan_report(comply_url,
+                                                   comply_password,
+                                                   username: username,
+                                                   status: status,
+                                                   ignorelist: ignorelist,
+                                                   onlylist: onlylist)
+        outfile = @data.fetch(:file, nil).nil? ? conf.fetch(:report_path, 'comply_scan_report.yaml') : @data[:file]
+        Abide::CLI::OUTPUT.yaml(report, file: outfile)
+      end
+    end
+  end
+end

data/lib/abide_dev_utils/cli/jira.rb

@@ -59,8 +59,8 @@ module Abide
       def execute(issue)
         client = JIRA.client(options: {})
         issue = client.Issue.find(issue)
-        console = @data[:file].nil? ? true : false
-        out_json = issue.attrs.select { |_,v| !v.nil? || !v.empty? }
+        console = @data[:file].nil?
+        out_json = issue.attrs.select { |_, v| !v.nil? || !v.empty? }
         Abide::CLI::OUTPUT.json(out_json, console: console, file: @data[:file])
       end
     end

data/lib/abide_dev_utils/comply.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'selenium-webdriver'
+require 'abide_dev_utils/output'
+
+module AbideDevUtils
+  module Comply
+    def self.scan_report(url, password, username: 'comply', status: nil, ignorelist: nil, onlylist: nil)
+      begin
+        AbideDevUtils::Output.simple 'Starting headless Chrome...'
+        options = Selenium::WebDriver::Chrome::Options.new
+        options.args = %w[
+          --headless
+          --test-type
+          --disable-gpu
+          --no-first-run
+          --no-default-browser-check
+          --ignore-certificate-errors
+          --start-maximized
+        ]
+        driver = Selenium::WebDriver.for :chrome, options: options
+        driver.get(url)
+        bypass_ssl_warning_page(driver)
+        AbideDevUtils::Output.simple "Logging into Comply at #{url}..."
+        login_to_comply(driver, username: username, password: password)
+        AbideDevUtils::Output.simple 'Finding nodes with scan reports...'
+        links = find_node_report_links(driver)
+        AbideDevUtils::Output.simple 'Building scan reports, this may take a while...'
+        build_report(driver, links, status: status, ignorelist: ignorelist, onlylist: onlylist)
+      ensure
+        driver.quit
+      end
+    end
+
+    def self.ignore_no_such_element
+      begin
+        yield
+      rescue Selenium::WebDriver::Error::NoSuchElementError => e
+        AbideDevUtils::Output.simple "Ignored exception #{e}", stream: $stderr
+      end
+    end
+
+    def self.wait_on(timeout = 10)
+      Selenium::WebDriver::Wait.new(timeout: timeout).until do
+        yield
+      end
+    end
+
+    def self.bypass_ssl_warning_page(driver)
+      ignore_no_such_element do
+        driver.find_element(id: 'details-button').click
+        driver.find_element(id: 'proceed-link').click
+      end
+    end
+
+    def self.login_to_comply(driver, username: 'comply', password: 'compliance')
+      wait_on { driver.find_element(id: 'username') }
+      driver.find_element(id: 'username').send_keys username
+      driver.find_element(id: 'password').send_keys password
+      driver.find_element(id: 'kc-login').click
+    end
+
+    def self.find_node_report_links(driver)
+      wait_on { driver.find_element(class: 'metric-containers-failed-hosts-count') }
+      hosts = driver.find_element(class: 'metric-containers-failed-hosts-count')
+      table = hosts.find_element(class: 'rc-table')
+      table_body = table.find_element(tag_name: 'tbody')
+      wait_on { table_body.find_element(tag_name: 'a') }
+      table_body.find_elements(tag_name: 'a')
+    end
+
+    def self.build_report(driver, links, status: nil, ignorelist: nil, onlylist: nil)
+      all_checks = {}
+      original_window = driver.window_handle
+      links.each do |link|
+        if !onlylist.nil? && !onlylist.empty?
+          next unless onlylist.include?(link.text)
+        elsif !ignorelist.nil? && !ignorelist.empty?
+          next if ignorelist.include?(link.text)
+        end
+        begin
+          node_name = link.text
+          progress = AbideDevUtils::Output.progress title: "Builingd report for #{node_name}", total: nil
+          link_url = link.attribute('href')
+          driver.manage.new_window(:tab)
+          wait_on { driver.window_handles.length == 2 }
+          progress.increment
+          driver.switch_to.window driver.window_handles[1]
+          driver.get(link_url)
+          wait_on { driver.find_element(class: 'details-scan-info') }
+          progress.increment
+          wait_on { driver.find_element(class: 'details-table') }
+          progress.increment
+          report = {}
+          report['scan_results'] = {}
+          scan_info_table = driver.find_element(class: 'details-scan-info')
+          scan_info_table_rows = scan_info_table.find_elements(tag_name: 'tr')
+          progress.increment
+          check_table_body = driver.find_element(tag_name: 'tbody')
+          check_table_rows = check_table_body.find_elements(tag_name: 'tr')
+          progress.increment
+          scan_info_table_rows.each do |row|
+            key = row.find_element(tag_name: 'h5').text
+            value = row.find_element(tag_name: 'strong').text
+            report[key.downcase.gsub(/:/, '').gsub(/ /, '_')] = value
+            progress.increment
+          end
+          check_table_rows.each do |row|
+            chk_objs = row.find_elements(tag_name: 'td')
+            chk_objs.map!(&:text)
+            if status.nil? || status.include?(chk_objs[1].downcase)
+              report['scan_results'][chk_objs[0][/^[0-9.]+/, 0]] = {
+                'name' => chk_objs[0].gsub(/\n/, ' '),
+                'status' => chk_objs[1]
+              }
+            end
+            progress.increment
+          end
+          all_checks[node_name] = report
+          driver.close
+          AbideDevUtils::Output.simple "Created report for #{node_name}"
+        ensure
+          driver.switch_to.window original_window
+        end
+      end
+      all_checks
+    end
+  end
+end

data/lib/abide_dev_utils/config.rb

@@ -13,6 +13,13 @@ module AbideDevUtils
       h.transform_keys(&:to_sym)
     end
 
+    def to_h(path = DEFAULT_PATH)
+      return {} unless File.file?(path)
+
+      h = YAML.safe_load(File.open(path), [Symbol])
+      h.transform_keys(&:to_sym)
+    end
+
     def self.config_section(section, path = DEFAULT_PATH)
       h = to_h(path)
       s = h.fetch(section.to_sym, nil)
@@ -21,8 +28,20 @@ module AbideDevUtils
       s.transform_keys(&:to_sym)
     end
 
+    def config_section(section, path = DEFAULT_PATH)
+      h = to_h(path)
+      s = h.fetch(section.to_sym, nil)
+      return {} if s.nil?
+
+      s.transform_keys(&:to_sym)
+    end
+
     def self.fetch(key, default = nil, path = DEFAULT_PATH)
       to_h(path).fetch(key, default)
     end
+
+    def fetch(key, default = nil, path = DEFAULT_PATH)
+      to_h(path).fetch(key, default)
+    end
   end
 end

data/lib/abide_dev_utils/jira.rb

@@ -135,6 +135,16 @@ module AbideDevUtils
       end
     end
 
+    # def self.new_issues_from_comply_report(client, project, report, dry_run: false)
+    #   dr_prefix = dry_run ? 'DRY RUN: ' : ''
+    #   i_attrs = all_project_issues_attrs(project)
+    #   rep_sums = summaries_from_coverage_report(report)
+    #   rep_sums.each do |k, v|
+    #     next if summary_exist?(k, i_attrs)
+
+    #     progress = AbideDevUtils::Output.progress(title: "#{dr_prefix}Creating Tasks", total: nil)
+    #     v.each do |s|
+
     def self.merge_options(options)
       config.merge(options)
     end
@@ -167,6 +177,11 @@ module AbideDevUtils
       summaries.transform_keys { |k| "#{COV_PARENT_SUMMARY_PREFIX}#{benchmark}-#{k}"}
     end
 
+    # def self.summaries_from_comply_report(report)
+    #   summaries = {}
+    #   report.each do |_, v|
+    # end
+
     class Dummy
       def attrs
         { 'fields' => {

data/lib/abide_dev_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AbideDevUtils
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: abide_dev_utils
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Heston Snodgrass
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-05-05 00:00:00.000000000 Z
+date: 2021-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.0.beta4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.0.beta4
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -275,10 +289,12 @@ files:
 - lib/abide_dev_utils.rb
 - lib/abide_dev_utils/cli.rb
 - lib/abide_dev_utils/cli/abstract.rb
+- lib/abide_dev_utils/cli/comply.rb
 - lib/abide_dev_utils/cli/jira.rb
 - lib/abide_dev_utils/cli/puppet.rb
 - lib/abide_dev_utils/cli/test.rb
 - lib/abide_dev_utils/cli/xccdf.rb
+- lib/abide_dev_utils/comply.rb
 - lib/abide_dev_utils/config.rb
 - lib/abide_dev_utils/constants.rb
 - lib/abide_dev_utils/errors.rb