abide_dev_utils 0.12.2 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e93a77cf4142b06ee4d41863ef2d0b737d21d37306b57082aa316f2731de5e9
-  data.tar.gz: 9b49e05aa575884d54efed8fd70b5662712845c083a8b59caeccf406a19f5633
+  metadata.gz: aa840c23e0e5ab5d978e04b6c3192bd473fa7d3886033be49c20124f33964c8b
+  data.tar.gz: 6d8d44b81c393907052d1cd70907b31e8e339ee7519080152ba3f2b5da3cd420
 SHA512:
-  metadata.gz: 51316ad752020f3b2297acbdc4bb328c5275d03e499721ee12381aaac5da3a6e77ecb9efeb0ffa4b3b6bb512154dbb4b36bf249f4816944a0110d3ceeb871b2b
-  data.tar.gz: 40b83d7be09ccb1349650617548276f6f34e266fe2e590e8c1b28b8e9c44f4499c93652abf9e254d77469fec9c98e9394a1ae2f4b2d3f0921827c3ce302dfb7e
+  metadata.gz: b18c4a042ae47492d235a3778f3b50a3589940af256cd0407ea38c170766fcaade5d8ead74c9baa47490c407bc9c375703356640e8f1e393a230055ddd099798
+  data.tar.gz: c2578428399f77596809838daa1adae12aa76e0f34298c3695bd9058a9da1a6c6e3a4e9120542dad9a6b2eda473581196e40df3467a8ff706332ce52824f2707

data/.github/workflows/mend_ruby.yaml

@@ -0,0 +1,39 @@
+name: mend_scan
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout repo content
+      uses: actions/checkout@v2 # checkout the repository content to github runner.
+      with:
+        fetch-depth: 1
+    - name: setup ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    - name: create lock
+      run: bundle lock
+    # install java
+    - uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin' # See 'Supported distributions' for available options
+        java-version: '17'
+    # download mend
+    - name: download_mend
+      run: curl -o wss-unified-agent.jar https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar
+    - name: run mend
+      run: java -jar wss-unified-agent.jar
+      env:
+        WS_APIKEY: ${{ secrets.MEND_API_KEY }}
+        WS_WSS_URL: https://saas-eu.whitesourcesoftware.com/agent
+        WS_USERKEY: ${{ secrets.MEND_TOKEN }}
+        WS_PRODUCTNAME: 'content-and-tooling' # I think this apply for our repo
+        WS_PROJECTNAME:  ${{ github.event.repository.name }}
+        WS_FILESYSTEMSCAN: true
+        WS_CHECKPOLICIES: true
+        WS_FORCEUPDATE: true

data/Gemfile.lock

@@ -1,15 +1,16 @@
 PATH
   remote: .
   specs:
-    abide_dev_utils (0.12.2)
+    abide_dev_utils (0.14.0)
       amatch (~> 0.4)
       cmdparse (~> 3.0)
-      facterdb (>= 1.18)
+      facterdb (>= 1.21)
       google-cloud-storage (~> 1.34)
       hashdiff (~> 1.0)
       jira-ruby (~> 2.2)
       nokogiri (~> 1.13)
       puppet (>= 6.23)
+      puppet-strings (>= 2.7)
       ruby-progressbar (~> 1.11)
       selenium-webdriver (~> 4.0.0.beta4)
 
@@ -17,7 +18,7 @@ GEM
   remote: https://rubygems.org/
   specs:
     CFPropertyList (2.3.6)
-    activesupport (7.0.3)
+    activesupport (7.0.4.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -60,10 +61,10 @@ GEM
     diff-lcs (1.5.0)
     digest-crc (0.6.4)
       rake (>= 12.0.0, < 14.0.0)
-    facter (4.2.11)
+    facter (4.2.14)
       hocon (~> 1.3)
       thor (>= 1.0.1, < 2.0)
-    facterdb (1.19.0)
+    facterdb (1.21.0)
       facter (< 5.0.0)
       jgrep
     faraday (2.3.0)
@@ -84,7 +85,7 @@ GEM
       octokit (~> 4.6)
       rainbow (>= 2.2.1)
       rake (>= 10.0)
-    google-apis-core (0.7.0)
+    google-apis-core (0.10.0)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (>= 0.16.2, < 2.a)
       httpclient (>= 2.8.1, < 3.a)
@@ -93,25 +94,25 @@ GEM
       retriable (>= 2.0, < 4.a)
       rexml
       webrick
-    google-apis-iamcredentials_v1 (0.13.0)
-      google-apis-core (>= 0.7, < 2.a)
-    google-apis-storage_v1 (0.17.0)
-      google-apis-core (>= 0.7, < 2.a)
+    google-apis-iamcredentials_v1 (0.16.0)
+      google-apis-core (>= 0.9.1, < 2.a)
+    google-apis-storage_v1 (0.19.0)
+      google-apis-core (>= 0.9.0, < 2.a)
     google-cloud-core (1.6.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
     google-cloud-env (1.6.0)
       faraday (>= 0.17.3, < 3.0)
-    google-cloud-errors (1.2.0)
-    google-cloud-storage (1.38.0)
+    google-cloud-errors (1.3.0)
+    google-cloud-storage (1.44.0)
       addressable (~> 2.8)
       digest-crc (~> 0.4)
       google-apis-iamcredentials_v1 (~> 0.1)
-      google-apis-storage_v1 (~> 0.17.0)
+      google-apis-storage_v1 (~> 0.19.0)
       google-cloud-core (~> 1.6)
       googleauth (>= 0.16.2, < 2.a)
       mini_mime (~> 1.0)
-    googleauth (1.2.0)
+    googleauth (1.3.0)
       faraday (>= 0.17.3, < 3.a)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
@@ -119,31 +120,34 @@ GEM
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
     hashdiff (1.0.1)
-    hiera (3.10.0)
+    hashie (5.0.0)
+    hiera (3.11.0)
     hocon (1.3.1)
     httpclient (2.8.3)
-    i18n (1.10.0)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     jgrep (1.5.4)
-    jira-ruby (2.2.0)
+    jira-ruby (2.3.0)
       activesupport
       atlassian-jwt
       multipart-post
       oauth (~> 0.5, >= 0.5.0)
-    jwt (2.4.1)
+    jwt (2.7.0)
     locale (2.1.3)
     memoist (0.16.2)
     method_source (1.0.0)
     mini_mime (1.1.2)
-    minitest (5.15.0)
-    mize (0.4.0)
+    minitest (5.17.0)
+    mize (0.4.1)
       protocol (~> 2.0)
     multi_json (1.15.0)
-    multipart-post (2.2.3)
+    multipart-post (2.3.0)
     nio4r (2.5.8)
-    nokogiri (1.13.8-x86_64-darwin)
+    nokogiri (1.14.1-x86_64-darwin)
       racc (~> 1.4)
-    oauth (0.5.10)
+    oauth (0.6.2)
+      snaky_hash (~> 2.0)
+      version_gem (~> 1.1)
     octokit (4.25.0)
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
@@ -164,9 +168,9 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (4.0.7)
-    puppet (7.18.0-universal-darwin)
+    puppet (7.22.0-universal-darwin)
       CFPropertyList (~> 2.2)
-      concurrent-ruby (~> 1.0)
+      concurrent-ruby (~> 1.0, < 1.2.0)
       deep_merge (~> 1.0)
       facter (> 2.0.1, < 5)
       fast_gettext (>= 1.1, < 3)
@@ -178,7 +182,10 @@ GEM
       semantic_puppet (~> 1.0)
     puppet-resource_api (1.8.14)
       hocon (>= 1.0)
-    racc (1.6.0)
+    puppet-strings (2.9.0)
+      rgen
+      yard (~> 0.9.5)
+    racc (1.6.2)
     rainbow (3.1.1)
     rake (13.0.6)
     regexp_parser (2.5.0)
@@ -188,6 +195,7 @@ GEM
       uber (< 0.2.0)
     retriable (3.1.2)
     rexml (3.2.5)
+    rgen (0.9.1)
     rspec (3.11.0)
       rspec-core (~> 3.11.0)
       rspec-expectations (~> 3.11.0)
@@ -221,7 +229,7 @@ GEM
       rubocop (~> 1.19)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
-    ruby_parser (3.19.1)
+    ruby_parser (3.19.2)
       sexp_processor (~> 4.16)
     rubyzip (2.3.2)
     sawyer (0.9.2)
@@ -239,18 +247,24 @@ GEM
       faraday (>= 0.17.5, < 3.a)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
+    snaky_hash (2.0.1)
+      hashie
+      version_gem (~> 1.1, >= 1.1.1)
     sync (0.5.0)
     thor (1.2.1)
     timers (4.3.3)
-    tins (1.31.1)
+    tins (1.32.1)
       sync
     traces (0.4.1)
     trailblazer-option (0.1.2)
-    tzinfo (2.0.4)
+    tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
     uber (0.1.0)
     unicode-display_width (2.1.0)
+    version_gem (1.1.1)
     webrick (1.7.0)
+    yard (0.9.28)
+      webrick (~> 1.7.0)
 
 PLATFORMS
   x86_64-darwin-19

data/abide_dev_utils.gemspec

@@ -35,13 +35,14 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'nokogiri', '~> 1.13'
   spec.add_dependency 'cmdparse', '~> 3.0'
   spec.add_dependency 'puppet', '>= 6.23'
+  spec.add_dependency 'puppet-strings', '>= 2.7'
   spec.add_dependency 'jira-ruby', '~> 2.2'
   spec.add_dependency 'ruby-progressbar', '~> 1.11'
   spec.add_dependency 'selenium-webdriver', '~> 4.0.0.beta4'
   spec.add_dependency 'google-cloud-storage', '~> 1.34'
   spec.add_dependency 'hashdiff', '~> 1.0'
   spec.add_dependency 'amatch', '~> 0.4'
-  spec.add_dependency 'facterdb', '>= 1.18'
+  spec.add_dependency 'facterdb', '>= 1.21'
 
   # Dev dependencies
   spec.add_development_dependency 'bundler'

data/lib/abide_dev_utils/cem/benchmark.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
 require 'set'
-require 'abide_dev_utils/dot_number_comparable'
-require 'abide_dev_utils/errors'
-require 'abide_dev_utils/ppt'
-require 'abide_dev_utils/cem/mapping/mapper'
+require_relative '../dot_number_comparable'
+require_relative '../errors'
+require_relative '../ppt'
+require_relative 'mapping/mapper'
 
 module AbideDevUtils
   module CEM
@@ -317,7 +317,8 @@ module AbideDevUtils
       def initialize(osname, major_version, hiera_conf, module_name, framework: 'cis')
         @osname = osname
         @major_version = major_version
-        @os_facts = AbideDevUtils::Ppt::FacterUtils.recursive_facts_for_os(@osname, @major_version)
+        @os_facts = AbideDevUtils::Ppt::FacterUtils::FactSets.new.find_by_fact_value_tuples(['os.name', @osname],
+                                                                                            ['os.release.major', @major_version])
         @osfamily = @os_facts['os']['family']
         @hiera_conf = hiera_conf
         @module_name = module_name
@@ -484,6 +485,8 @@ module AbideDevUtils
         raise AbideDevUtils::Errors::ResourceDataNotFoundError, facts if rdata_files.nil? || rdata_files.empty?
 
         YAML.load_file(rdata_files[0].path)
+      rescue StandardError => e
+        require 'pry'; binding.pry
       end
     end
   end

data/lib/abide_dev_utils/cem/generate/coverage_report.rb

@@ -15,14 +15,12 @@ module AbideDevUtils
       # the various compliance frameworks expect to be enforced.
       module CoverageReport
         def self.generate(format_func: :to_h, opts: {})
-          opts = AbideDevUtils::CEM::CoverageReport::ReportOptions.new(opts)
+          opts = ReportOptions.new(opts)
           pupmod = AbideDevUtils::Ppt::PuppetModule.new
           benchmarks = AbideDevUtils::CEM::Benchmark.benchmarks_from_puppet_module(pupmod,
-                                                                                  ignore_all_errors: opts.ignore_all_errors)
+                                                                                   ignore_all_errors: opts.ignore_all_errors)
           benchmarks.map do |b|
-            AbideDevUtils::CEM::CoverageReport::BenchmarkReport.new(b)
-                                                              .send(report_type, **{ profile: profile, level: level })
-                                                              .send(format_func)
+            BenchmarkReport.new(b, opts).run.send(format_func)
           end
         end
 
@@ -306,11 +304,15 @@ module AbideDevUtils
 
         # Creates ReportOutput objects based on the given Benchmark
         class BenchmarkReport
-          def initialize(benchmark, opts = AbideDevUtils::CEM::CoverageReport::ReportOptions.new)
+          def initialize(benchmark, opts = ReportOptions.new)
             @benchmark = benchmark
             @opts = opts
           end
 
+          def run
+            send(@opts.report_type)
+          end
+
           def controls_in_resource_data
             @controls_in_resource_data ||= find_controls_in_resource_data
           end
@@ -322,7 +324,7 @@ module AbideDevUtils
           def basic_coverage(level: @opts.level, profile: @opts.profile)
             map_type = @benchmark.map_type(controls_in_resource_data[0])
             rules_in_map = @benchmark.rules_in_map(map_type, level: level, profile: profile)
-            AbideDevUtils::CEM::CoverageReport::ReportOutput.new(@benchmark, controls_in_resource_data, rules_in_map)
+            ReportOutput.new(@benchmark, controls_in_resource_data, rules_in_map)
           end
 
           # def correlated_coverage(level: @opts.level, profile: @opts.profile)

data/lib/abide_dev_utils/cem/generate/reference.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'json'
+require 'puppet-strings'
+require 'puppet-strings/yard'
 require 'shellwords'
 require 'timeout'
 require 'yaml'
@@ -31,7 +33,7 @@ module AbideDevUtils
           case data.fetch(:format, 'markdown')
           when 'markdown'
             file = data[:out_file] || 'REFERENCE.md'
-            MarkdownGenerator.new(benchmarks, pupmod.name, file: file).generate(doc_title)
+            MarkdownGenerator.new(benchmarks, pupmod.name, file: file, opts: data).generate(doc_title)
           else
             raise "Format #{data[:format]} is unsupported! Only `markdown` format supported"
           end
@@ -61,31 +63,35 @@ module AbideDevUtils
         class MarkdownGenerator
           SPECIAL_CONTROL_IDS = %w[dependent cem_options cem_protected].freeze
 
-          def initialize(benchmarks, module_name, file: 'REFERENCE.md')
+          def initialize(benchmarks, module_name, file: 'REFERENCE.md', opts: {})
             @benchmarks = benchmarks
             @module_name = module_name
             @file = file
+            @opts = opts
             @md = AbideDevUtils::Markdown.new(@file)
           end
 
           def generate(doc_title = 'Reference')
+            @strings = Strings.new(opts: @opts)
             md.add_title(doc_title)
             benchmarks.each do |benchmark|
-              progress_bar = AbideDevUtils::Output.progress(title: "Generating Markdown for #{benchmark.title_key}",
-                                                            total: benchmark.controls.length)
+              unless @opts[:quiet]
+                progress_bar = AbideDevUtils::Output.progress(title: "Generating Markdown for #{benchmark.title_key}",
+                                                              total: benchmark.controls.length)
+              end
               md.add_h1(benchmark.title_key)
               benchmark.controls.each do |control|
                 next if SPECIAL_CONTROL_IDS.include? control.id
                 next if benchmark.framework == 'stig' && control.id_map_type != 'vulnid'
 
-                control_md = ControlMarkdown.new(control, @md, @module_name, benchmark.framework)
+                control_md = ControlMarkdown.new(control, @md, @strings, @module_name, benchmark.framework, opts: @opts)
                 control_md.generate!
-                progress_bar.increment
+                progress_bar.increment unless @opts[:quiet]
               rescue StandardError => e
                 raise "Failed to generate markdown for control #{control.id}. Original message: #{e.message}"
               end
             end
-            AbideDevUtils::Output.simple("Saving markdown to #{@file}")
+            AbideDevUtils::Output.simple("Saving markdown to #{@file}") unless @opts[:quiet]
             md.to_file
           end
 
@@ -96,13 +102,152 @@ module AbideDevUtils
 
         class ConfigExampleError < StandardError; end
 
+        # Puppet Strings reference object
+        class Strings
+          REGISTRY_TYPES = %i[
+            root
+            module
+            class
+            puppet_class
+            puppet_data_type
+            puppet_data_type_alias
+            puppet_defined_type
+            puppet_type
+            puppet_provider
+            puppet_function
+            puppet_task
+            puppet_plan
+          ].freeze
+
+          attr_reader :search_patterns
+
+          def initialize(search_patterns: nil, opts: {})
+            @search_patterns = search_patterns || PuppetStrings::DEFAULT_SEARCH_PATTERNS
+            @debug = opts[:debug]
+            @quiet = opts[:quiet]
+            PuppetStrings::Yard.setup!
+            YARD::CLI::Yardoc.run(*yard_args(@search_patterns, debug: @debug, quiet: @quiet))
+          end
+
+          def debug?
+            !!@debug
+          end
+
+          def quiet?
+            !!@quiet
+          end
+
+          def registry
+            @registry ||= YARD::Registry.all(*REGISTRY_TYPES)
+          end
+
+          def find_resource(resource_name)
+            to_h.each do |_, resources|
+              res = resources.find { |r| r[:name] == resource_name.to_sym }
+              return res if res
+            end
+          end
+
+          def puppet_classes
+            @puppet_classes ||= hashes_for_reg_type(:puppet_class)
+          end
+
+          def data_types
+            @data_types ||= hashes_for_reg_type(:puppet_data_types)
+          end
+
+          def data_type_aliases
+            @data_type_aliases ||= hashes_for_reg_type(:puppet_data_type_alias)
+          end
+
+          def defined_types
+            @defined_types ||= hashes_for_reg_type(:puppet_defined_type)
+          end
+
+          def resource_types
+            @resource_types ||= hashes_for_reg_type(:puppet_type)
+          end
+
+          def providers
+            @providers ||= hashes_for_reg_type(:puppet_provider)
+          end
+
+          def puppet_functions
+            @puppet_functions ||= hashes_for_reg_type(:puppet_function)
+          end
+
+          def puppet_tasks
+            @puppet_tasks ||= hashes_for_reg_type(:puppet_task)
+          end
+
+          def puppet_plans
+            @puppet_plans ||= hashes_for_reg_type(:puppet_plan)
+          end
+
+          def to_h
+            {
+              puppet_classes: puppet_classes,
+              data_types: data_types,
+              data_type_aliases: data_type_aliases,
+              defined_types: defined_types,
+              resource_types: resource_types,
+              providers: providers,
+              puppet_functions: puppet_functions,
+              puppet_tasks: puppet_tasks,
+              puppet_plans: puppet_plans,
+            }
+          end
+
+          private
+
+          def hashes_for_reg_type(reg_type)
+            all_to_h(registry.select { |i| i.type == reg_type })
+          end
+
+          def all_to_h(objects)
+            objects.sort_by(&:name).map(&:to_hash)
+          end
+
+          def yard_args(patterns, debug: false, quiet: false)
+            args = ['doc', '--no-progress', '-n']
+            args << '--debug' if debug && !quiet
+            args << '--backtrace' if debug && !quiet
+            args << '-q' if quiet
+            args << '--no-stats' if quiet
+            args += patterns
+            args
+          end
+        end
+
+        # Generates markdown for Puppet classes based on Puppet Strings JSON
+        # class PuppetClassMarkdown
+        #   def initialize(puppet_classes, md, opts: {})
+        #     @puppet_classes = puppet_classes
+        #     @md = md
+        #     @opts = opts
+        #   end
+
+        #   def generate!
+        #     @puppet_classes.each do |puppet_class|
+        #       @md.add_h2(puppet_class['name'])
+        #       @md.add_paragraph("File(Line): `#{puppet_class['file']}(#{puppet_class['line']})`")
+
+        #   private
+
+        #   def doc_string_builder(puppet_class)
+        #     return if puppet_class['docstring'].nil? || puppet_class['docstring'].empty?
+        # end
+
+        # Generates markdown for a control
         class ControlMarkdown
-          def initialize(control, md, module_name, framework, formatter: nil)
+          def initialize(control, md, strings, module_name, framework, formatter: nil, opts: {})
             @control = control
             @md = md
+            @strings = strings
             @module_name = module_name
             @framework = framework
             @formatter = formatter.nil? ? TypeExprValueFormatter : formatter
+            @opts = opts
             @control_data = {}
           end
 
@@ -156,6 +301,27 @@ module AbideDevUtils
             " - #{@md.italic('Default:')} #{@md.code(@control_data[ctrl_param[:name]][:default])}"
           end
 
+          def param_description(ctrl_param)
+            res = if @control.resource.type == 'class'
+                    @strings.find_resource(@control.resource.title)
+                  else
+                    @strings.find_resource(@control.resource.type)
+                  end
+            return unless res&.key?(:docstring) && res[:docstring].key?(:tags)
+            return if res[:docstring][:tags].empty? || res[:docstring][:tags].none? { |x| x[:tag_name] == 'param' }
+
+            param_tag = res[:docstring][:tags].find { |x| x[:tag_name] == 'param' && x[:name] == ctrl_param[:name] }
+            if param_tag.nil? || param_tag[:text].nil? || param_tag[:text].chomp.empty?
+              if @opts[:strict]
+                raise "No description found for parameter #{ctrl_param[:name]} in resource #{@control.resource.title}"
+              end
+
+              return
+            end
+
+            " - #{param_tag[:text]}"
+          end
+
           def control_params_builder
             return unless control_has_valid_params?
 
@@ -164,6 +330,8 @@ module AbideDevUtils
               collection.each do |hsh|
                 rparam = resource_param(hsh)
                 str_array = [@md.code(hsh[:name]), param_type_expr(hsh, rparam), param_default_value(hsh, rparam)]
+                desc = param_description(hsh)
+                str_array << desc if desc
                 @md.add_ul(str_array.compact.join, indent: 1)
               end
             end