abide_dev_utils 0.13.0 → 0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 863b7f1da9e228c425708434e5adb149a04462c6f9e7a0189799761d9420b6e1
-  data.tar.gz: c221c833c4f6b89d80328b3e3bf2de527bb400fe8df5946af2c46d5be45d1b60
+  metadata.gz: 54b60da7a8a67908351bf5e2687c39e8ac76aad00cb71ad798ac0c0a195e07fe
+  data.tar.gz: 400a2593d83b24a0f970a52f4b6ae0434ff41975260b351c5dd5d0e52387f282
 SHA512:
-  metadata.gz: 715413f107df1c3f52269658b124db9bad5b4113042ca9e18eb11cabd73cb3b83a57e3f2adf13591fc168dc841be1fb93d3e972ce74aafe6888b95802b644316
-  data.tar.gz: 10eed44e5eb6969974f3426cc65c6c7a00cc988fc5a8ef733df50fafd1c395df44d7afec4b18f51c0ae42d5a36204174a88c42679a6eb3616080deccfb38a86f
+  metadata.gz: 3857661d49e0fd254fda83c58ac0363cad5df50342aa55eca44f05f8e1dd47214200ef8586bbec6c1f46767cd7d0528be8f089507e0a6699896971ffceaca2da
+  data.tar.gz: edaa3a7dc991292d51feba476cd49cf343e570ac87e0310957f4d55d43932fd3dc4ed6d72aa6f78fbc686063416e65ed0bbed03bf550fffcf63a135c3175d4af

data/.github/workflows/mend_ruby.yaml

@@ -0,0 +1,39 @@
+name: mend_scan
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout repo content
+      uses: actions/checkout@v2 # checkout the repository content to github runner.
+      with:
+        fetch-depth: 1
+    - name: setup ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    - name: create lock
+      run: bundle lock
+    # install java
+    - uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin' # See 'Supported distributions' for available options
+        java-version: '17'
+    # download mend
+    - name: download_mend
+      run: curl -o wss-unified-agent.jar https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar
+    - name: run mend
+      run: java -jar wss-unified-agent.jar
+      env:
+        WS_APIKEY: ${{ secrets.MEND_API_KEY }}
+        WS_WSS_URL: https://saas-eu.whitesourcesoftware.com/agent
+        WS_USERKEY: ${{ secrets.MEND_TOKEN }}
+        WS_PRODUCTNAME: 'content-and-tooling' # I think this apply for our repo
+        WS_PROJECTNAME:  ${{ github.event.repository.name }}
+        WS_FILESYSTEMSCAN: true
+        WS_CHECKPOLICIES: true
+        WS_FORCEUPDATE: true

data/Gemfile.lock

@@ -1,15 +1,16 @@
 PATH
   remote: .
   specs:
-    abide_dev_utils (0.13.0)
+    abide_dev_utils (0.14.1)
       amatch (~> 0.4)
       cmdparse (~> 3.0)
-      facterdb (>= 1.18)
+      facterdb (>= 1.21)
       google-cloud-storage (~> 1.34)
       hashdiff (~> 1.0)
       jira-ruby (~> 2.2)
       nokogiri (~> 1.13)
       puppet (>= 6.23)
+      puppet-strings (>= 2.7)
       ruby-progressbar (~> 1.11)
       selenium-webdriver (~> 4.0.0.beta4)
 
@@ -144,8 +145,6 @@ GEM
     nio4r (2.5.8)
     nokogiri (1.14.1-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.1-x86_64-linux)
-      racc (~> 1.4)
     oauth (0.6.2)
       snaky_hash (~> 2.0)
       version_gem (~> 1.1)
@@ -169,17 +168,6 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (4.0.7)
-    puppet (7.22.0)
-      concurrent-ruby (~> 1.0, < 1.2.0)
-      deep_merge (~> 1.0)
-      facter (> 2.0.1, < 5)
-      fast_gettext (>= 1.1, < 3)
-      hiera (>= 3.2.1, < 4)
-      locale (~> 2.1)
-      multi_json (~> 1.10)
-      puppet-resource_api (~> 1.5)
-      scanf (~> 1.0)
-      semantic_puppet (~> 1.0)
     puppet (7.22.0-universal-darwin)
       CFPropertyList (~> 2.2)
       concurrent-ruby (~> 1.0, < 1.2.0)
@@ -194,6 +182,9 @@ GEM
       semantic_puppet (~> 1.0)
     puppet-resource_api (1.8.14)
       hocon (>= 1.0)
+    puppet-strings (2.9.0)
+      rgen
+      yard (~> 0.9.5)
     racc (1.6.2)
     rainbow (3.1.1)
     rake (13.0.6)
@@ -204,6 +195,7 @@ GEM
       uber (< 0.2.0)
     retriable (3.1.2)
     rexml (3.2.5)
+    rgen (0.9.1)
     rspec (3.11.0)
       rspec-core (~> 3.11.0)
       rspec-expectations (~> 3.11.0)
@@ -270,7 +262,9 @@ GEM
     uber (0.1.0)
     unicode-display_width (2.1.0)
     version_gem (1.1.1)
-    webrick (1.8.1)
+    webrick (1.7.0)
+    yard (0.9.28)
+      webrick (~> 1.7.0)
 
 PLATFORMS
   x86_64-darwin-19

data/abide_dev_utils.gemspec

@@ -35,13 +35,14 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'nokogiri', '~> 1.13'
   spec.add_dependency 'cmdparse', '~> 3.0'
   spec.add_dependency 'puppet', '>= 6.23'
+  spec.add_dependency 'puppet-strings', '>= 2.7'
   spec.add_dependency 'jira-ruby', '~> 2.2'
   spec.add_dependency 'ruby-progressbar', '~> 1.11'
   spec.add_dependency 'selenium-webdriver', '~> 4.0.0.beta4'
   spec.add_dependency 'google-cloud-storage', '~> 1.34'
   spec.add_dependency 'hashdiff', '~> 1.0'
   spec.add_dependency 'amatch', '~> 0.4'
-  spec.add_dependency 'facterdb', '>= 1.18'
+  spec.add_dependency 'facterdb', '>= 1.21'
 
   # Dev dependencies
   spec.add_development_dependency 'bundler'

data/lib/abide_dev_utils/cem/benchmark.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
 require 'set'
-require 'abide_dev_utils/dot_number_comparable'
-require 'abide_dev_utils/errors'
-require 'abide_dev_utils/ppt'
-require 'abide_dev_utils/cem/mapping/mapper'
+require_relative '../dot_number_comparable'
+require_relative '../errors'
+require_relative '../ppt'
+require_relative 'mapping/mapper'
 
 module AbideDevUtils
   module CEM
@@ -317,7 +317,8 @@ module AbideDevUtils
       def initialize(osname, major_version, hiera_conf, module_name, framework: 'cis')
         @osname = osname
         @major_version = major_version
-        @os_facts = AbideDevUtils::Ppt::FacterUtils.recursive_facts_for_os(@osname, @major_version)
+        @os_facts = AbideDevUtils::Ppt::FacterUtils::FactSets.new.find_by_fact_value_tuples(['os.name', @osname],
+                                                                                            ['os.release.major', @major_version])
         @osfamily = @os_facts['os']['family']
         @hiera_conf = hiera_conf
         @module_name = module_name
@@ -484,6 +485,8 @@ module AbideDevUtils
         raise AbideDevUtils::Errors::ResourceDataNotFoundError, facts if rdata_files.nil? || rdata_files.empty?
 
         YAML.load_file(rdata_files[0].path)
+      rescue StandardError => e
+        require 'pry'; binding.pry
       end
     end
   end

data/lib/abide_dev_utils/cem/generate/coverage_report.rb

@@ -15,14 +15,12 @@ module AbideDevUtils
       # the various compliance frameworks expect to be enforced.
       module CoverageReport
         def self.generate(format_func: :to_h, opts: {})
-          opts = AbideDevUtils::CEM::CoverageReport::ReportOptions.new(opts)
+          opts = ReportOptions.new(opts)
           pupmod = AbideDevUtils::Ppt::PuppetModule.new
           benchmarks = AbideDevUtils::CEM::Benchmark.benchmarks_from_puppet_module(pupmod,
-                                                                                  ignore_all_errors: opts.ignore_all_errors)
+                                                                                   ignore_all_errors: opts.ignore_all_errors)
           benchmarks.map do |b|
-            AbideDevUtils::CEM::CoverageReport::BenchmarkReport.new(b)
-                                                              .send(report_type, **{ profile: profile, level: level })
-                                                              .send(format_func)
+            BenchmarkReport.new(b, opts).run.send(format_func)
           end
         end
 
@@ -306,11 +304,15 @@ module AbideDevUtils
 
         # Creates ReportOutput objects based on the given Benchmark
         class BenchmarkReport
-          def initialize(benchmark, opts = AbideDevUtils::CEM::CoverageReport::ReportOptions.new)
+          def initialize(benchmark, opts = ReportOptions.new)
             @benchmark = benchmark
             @opts = opts
           end
 
+          def run
+            send(@opts.report_type)
+          end
+
           def controls_in_resource_data
             @controls_in_resource_data ||= find_controls_in_resource_data
           end
@@ -322,7 +324,7 @@ module AbideDevUtils
           def basic_coverage(level: @opts.level, profile: @opts.profile)
             map_type = @benchmark.map_type(controls_in_resource_data[0])
             rules_in_map = @benchmark.rules_in_map(map_type, level: level, profile: profile)
-            AbideDevUtils::CEM::CoverageReport::ReportOutput.new(@benchmark, controls_in_resource_data, rules_in_map)
+            ReportOutput.new(@benchmark, controls_in_resource_data, rules_in_map)
           end
 
           # def correlated_coverage(level: @opts.level, profile: @opts.profile)

data/lib/abide_dev_utils/cem/generate/reference.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'json'
+require 'puppet-strings'
+require 'puppet-strings/yard'
 require 'shellwords'
 require 'timeout'
 require 'yaml'
@@ -31,7 +33,7 @@ module AbideDevUtils
           case data.fetch(:format, 'markdown')
           when 'markdown'
             file = data[:out_file] || 'REFERENCE.md'
-            MarkdownGenerator.new(benchmarks, pupmod.name, file: file).generate(doc_title)
+            MarkdownGenerator.new(benchmarks, pupmod.name, file: file, opts: data).generate(doc_title)
           else
             raise "Format #{data[:format]} is unsupported! Only `markdown` format supported"
           end
@@ -61,31 +63,35 @@ module AbideDevUtils
         class MarkdownGenerator
           SPECIAL_CONTROL_IDS = %w[dependent cem_options cem_protected].freeze
 
-          def initialize(benchmarks, module_name, file: 'REFERENCE.md')
+          def initialize(benchmarks, module_name, file: 'REFERENCE.md', opts: {})
             @benchmarks = benchmarks
             @module_name = module_name
             @file = file
+            @opts = opts
             @md = AbideDevUtils::Markdown.new(@file)
           end
 
           def generate(doc_title = 'Reference')
+            @strings = Strings.new(opts: @opts)
             md.add_title(doc_title)
             benchmarks.each do |benchmark|
-              progress_bar = AbideDevUtils::Output.progress(title: "Generating Markdown for #{benchmark.title_key}",
-                                                            total: benchmark.controls.length)
+              unless @opts[:quiet]
+                progress_bar = AbideDevUtils::Output.progress(title: "Generating Markdown for #{benchmark.title_key}",
+                                                              total: benchmark.controls.length)
+              end
               md.add_h1(benchmark.title_key)
               benchmark.controls.each do |control|
                 next if SPECIAL_CONTROL_IDS.include? control.id
                 next if benchmark.framework == 'stig' && control.id_map_type != 'vulnid'
 
-                control_md = ControlMarkdown.new(control, @md, @module_name, benchmark.framework)
+                control_md = ControlMarkdown.new(control, @md, @strings, @module_name, benchmark.framework, opts: @opts)
                 control_md.generate!
-                progress_bar.increment
+                progress_bar.increment unless @opts[:quiet]
               rescue StandardError => e
                 raise "Failed to generate markdown for control #{control.id}. Original message: #{e.message}"
               end
             end
-            AbideDevUtils::Output.simple("Saving markdown to #{@file}")
+            AbideDevUtils::Output.simple("Saving markdown to #{@file}") unless @opts[:quiet]
             md.to_file
           end
 
@@ -96,13 +102,152 @@ module AbideDevUtils
 
         class ConfigExampleError < StandardError; end
 
+        # Puppet Strings reference object
+        class Strings
+          REGISTRY_TYPES = %i[
+            root
+            module
+            class
+            puppet_class
+            puppet_data_type
+            puppet_data_type_alias
+            puppet_defined_type
+            puppet_type
+            puppet_provider
+            puppet_function
+            puppet_task
+            puppet_plan
+          ].freeze
+
+          attr_reader :search_patterns
+
+          def initialize(search_patterns: nil, opts: {})
+            @search_patterns = search_patterns || PuppetStrings::DEFAULT_SEARCH_PATTERNS
+            @debug = opts[:debug]
+            @quiet = opts[:quiet]
+            PuppetStrings::Yard.setup!
+            YARD::CLI::Yardoc.run(*yard_args(@search_patterns, debug: @debug, quiet: @quiet))
+          end
+
+          def debug?
+            !!@debug
+          end
+
+          def quiet?
+            !!@quiet
+          end
+
+          def registry
+            @registry ||= YARD::Registry.all(*REGISTRY_TYPES)
+          end
+
+          def find_resource(resource_name)
+            to_h.each do |_, resources|
+              res = resources.find { |r| r[:name] == resource_name.to_sym }
+              return res if res
+            end
+          end
+
+          def puppet_classes
+            @puppet_classes ||= hashes_for_reg_type(:puppet_class)
+          end
+
+          def data_types
+            @data_types ||= hashes_for_reg_type(:puppet_data_types)
+          end
+
+          def data_type_aliases
+            @data_type_aliases ||= hashes_for_reg_type(:puppet_data_type_alias)
+          end
+
+          def defined_types
+            @defined_types ||= hashes_for_reg_type(:puppet_defined_type)
+          end
+
+          def resource_types
+            @resource_types ||= hashes_for_reg_type(:puppet_type)
+          end
+
+          def providers
+            @providers ||= hashes_for_reg_type(:puppet_provider)
+          end
+
+          def puppet_functions
+            @puppet_functions ||= hashes_for_reg_type(:puppet_function)
+          end
+
+          def puppet_tasks
+            @puppet_tasks ||= hashes_for_reg_type(:puppet_task)
+          end
+
+          def puppet_plans
+            @puppet_plans ||= hashes_for_reg_type(:puppet_plan)
+          end
+
+          def to_h
+            {
+              puppet_classes: puppet_classes,
+              data_types: data_types,
+              data_type_aliases: data_type_aliases,
+              defined_types: defined_types,
+              resource_types: resource_types,
+              providers: providers,
+              puppet_functions: puppet_functions,
+              puppet_tasks: puppet_tasks,
+              puppet_plans: puppet_plans,
+            }
+          end
+
+          private
+
+          def hashes_for_reg_type(reg_type)
+            all_to_h(registry.select { |i| i.type == reg_type })
+          end
+
+          def all_to_h(objects)
+            objects.sort_by(&:name).map(&:to_hash)
+          end
+
+          def yard_args(patterns, debug: false, quiet: false)
+            args = ['doc', '--no-progress', '-n']
+            args << '--debug' if debug && !quiet
+            args << '--backtrace' if debug && !quiet
+            args << '-q' if quiet
+            args << '--no-stats' if quiet
+            args += patterns
+            args
+          end
+        end
+
+        # Generates markdown for Puppet classes based on Puppet Strings JSON
+        # class PuppetClassMarkdown
+        #   def initialize(puppet_classes, md, opts: {})
+        #     @puppet_classes = puppet_classes
+        #     @md = md
+        #     @opts = opts
+        #   end
+
+        #   def generate!
+        #     @puppet_classes.each do |puppet_class|
+        #       @md.add_h2(puppet_class['name'])
+        #       @md.add_paragraph("File(Line): `#{puppet_class['file']}(#{puppet_class['line']})`")
+
+        #   private
+
+        #   def doc_string_builder(puppet_class)
+        #     return if puppet_class['docstring'].nil? || puppet_class['docstring'].empty?
+        # end
+
+        # Generates markdown for a control
         class ControlMarkdown
-          def initialize(control, md, module_name, framework, formatter: nil)
+          def initialize(control, md, strings, module_name, framework, formatter: nil, opts: {})
             @control = control
             @md = md
+            @strings = strings
             @module_name = module_name
             @framework = framework
             @formatter = formatter.nil? ? TypeExprValueFormatter : formatter
+            @opts = opts
             @control_data = {}
           end
 
@@ -156,6 +301,27 @@ module AbideDevUtils
             " - #{@md.italic('Default:')} #{@md.code(@control_data[ctrl_param[:name]][:default])}"
           end
 
+          def param_description(ctrl_param)
+            res = if @control.resource.type == 'class'
+                    @strings.find_resource(@control.resource.title)
+                  else
+                    @strings.find_resource(@control.resource.type)
+                  end
+            return unless res&.key?(:docstring) && res[:docstring].key?(:tags)
+            return if res[:docstring][:tags].empty? || res[:docstring][:tags].none? { |x| x[:tag_name] == 'param' }
+
+            param_tag = res[:docstring][:tags].find { |x| x[:tag_name] == 'param' && x[:name] == ctrl_param[:name] }
+            if param_tag.nil? || param_tag[:text].nil? || param_tag[:text].chomp.empty?
+              if @opts[:strict]
+                raise "No description found for parameter #{ctrl_param[:name]} in resource #{@control.resource.title}"
+              end
+
+              return
+            end
+
+            " - #{param_tag[:text]}"
+          end
+
           def control_params_builder
             return unless control_has_valid_params?
 
@@ -164,6 +330,8 @@ module AbideDevUtils
               collection.each do |hsh|
                 rparam = resource_param(hsh)
                 str_array = [@md.code(hsh[:name]), param_type_expr(hsh, rparam), param_default_value(hsh, rparam)]
+                desc = param_description(hsh)
+                str_array << desc if desc
                 @md.add_ul(str_array.compact.join, indent: 1)
               end
             end

data/lib/abide_dev_utils/cem/validate/strings/base_validator.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require_relative 'validation_finding'
+
+module AbideDevUtils
+  module CEM
+    module Validate
+      module Strings
+        # Base class for validating Puppet Strings objects. This class can be used directly, but it is
+        # recommended to use a subclass of this class to provide more specific validation logic. Each
+        # subclass should implement a `validate_<type>` method that will be called by the `validate` method
+        # of this class. The `validate_<type>` method should contain the validation logic for the
+        # corresponding type of Puppet Strings object.
+        class BaseValidator
+          SAFE_OBJECT_METHODS = %i[
+            type
+            name
+            file
+            line
+            docstring
+            tags
+            parameters
+            source
+          ].freeze
+          PDK_SUMMARY_REGEX = %r{^A short summary of the purpose.*}.freeze
+          PDK_DESCRIPTION_REGEX = %r{^A description of what this.*}.freeze
+
+          attr_reader :findings
+
+          def initialize(strings_object)
+            @object = strings_object
+            @findings = []
+            # Define instance methods for each of the SAFE_OBJECT_METHODS
+            SAFE_OBJECT_METHODS.each do |method|
+              define_singleton_method(method) { safe_method_call(@object, method) }
+            end
+          end
+
+          def errors
+            @findings.select { |f| f.type == :error }
+          end
+
+          def warnings
+            @findings.select { |f| f.type == :warning }
+          end
+
+          def errors?
+            !errors.empty?
+          end
+
+          def warnings?
+            !warnings.empty?
+          end
+
+          def find_tag_name(tag_name)
+            tags&.find { |t| t.tag_name == tag_name }
+          end
+
+          def select_tag_name(tag_name)
+            return [] if tags.nil?
+
+            tags.select { |t| t.tag_name == tag_name }
+          end
+
+          def find_parameter(param_name)
+            parameters&.find { |p| p[0] == param_name }
+          end
+
+          def validate
+            license_tag?
+            non_generic_summary?
+            non_generic_description?
+            send("validate_#{type}".to_sym) if respond_to?("validate_#{type}".to_sym)
+          end
+
+          # Checks if the object has a license tag and if it is formatted correctly.
+          # Comparison is not case sensitive.
+          def license_tag?
+            see_tags = select_tag_name('see')
+            if see_tags.empty? || see_tags.none? { |t| t.name.casecmp('LICENSE.pdf') && t.text.casecmp('for license') }
+              new_finding(
+                :error,
+                :no_license_tag,
+                remediation: 'Add "@see LICENSE.pdf for license" to the class documentation'
+              )
+              return false
+            end
+            true
+          end
+
+          # Checks if the summary is not the default PDK summary.
+          def non_generic_summary?
+            summary = find_tag_name('summary')&.text
+            return true if summary.nil?
+
+            if summary.match?(PDK_SUMMARY_REGEX)
+              new_finding(:warning, :generic_summary, remediation: 'Add a more descriptive summary')
+              return false
+            end
+            true
+          end
+
+          # Checks if the description is not the default PDK description.
+          def non_generic_description?
+            description = docstring
+            return true if description.nil?
+
+            if description.match?(PDK_DESCRIPTION_REGEX)
+              new_finding(:warning, :generic_description, remediation: 'Add a more descriptive description')
+              return false
+            end
+            true
+          end
+
+          private
+
+          def safe_method_call(obj, method, *args)
+            obj.send(method, *args)
+          rescue NoMethodError
+            nil
+          end
+
+          def new_finding(type, title, **data)
+            @findings << ValidationFinding.new(type, title.to_sym, data)
+          end
+        end
+      end
+    end
+  end
+end