abide_dev_utils 0.9.7 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc24decc17e394464d7ccfe4b461e8e6a5e74792b3a6a76092d5bc8d1fc9d87f
-  data.tar.gz: 43811951c377e43d3e6a10b9d295e807472d17b37ddda143d1a7e3f39a277a6c
+  metadata.gz: 4fc3820895d385ca28d6e097c1822cfaf878d41e2b282bfca55bec2e884a2dc2
+  data.tar.gz: de27ab8fb2021de5cd437695566e4ab0ad0531bc94659948557b3b09a05d59f5
 SHA512:
-  metadata.gz: 8228550f80f8234306b6f47ec45d4abd62df2953826b40cd225f8431753f1d72eff7e5c4caec2d5adedeac45d77c495116e606d3d8e66cfb2f662d92306e3f79
-  data.tar.gz: 7bd968ca178a24897184997445d36ece5e6c2bcbdd0a3bed9156b984c58cce1b4e289e9e8894a71d3a3ccab0d02c25a88b2b5005190db97e49f40173c0ed6479
+  metadata.gz: 7df796781a8f4e4b87cf324c83273a9d2964f44f8dc8fd9f68c65610295f7b9f5161b70c23d66ab19ff75ca610860d27245256dabf6e69633850cbd2d1f3dd96
+  data.tar.gz: 3cf0142c1a0f3fae3bab07f65add4a320f90ca900adc5be63b77f72e40edba4936b6ea835796d0fd6b2c3cff034b6a2dcb4f5bd1ff43a476b40bcc053c5356cb

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    abide_dev_utils (0.9.7)
+    abide_dev_utils (0.10.0)
+      amatch (~> 0.4)
       cmdparse (~> 3.0)
       google-cloud-storage (~> 1.34)
       hashdiff (~> 1.0)
@@ -21,6 +22,9 @@ GEM
       tzinfo (~> 2.0)
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
+    amatch (0.4.0)
+      mize
+      tins (~> 1.0)
     ast (2.4.2)
     async (1.30.1)
       console (~> 1.10)
@@ -53,7 +57,7 @@ GEM
     diff-lcs (1.5.0)
     digest-crc (0.6.4)
       rake (>= 12.0.0, < 14.0.0)
-    facter (4.2.7)
+    facter (4.2.9)
       hocon (~> 1.3)
       thor (>= 1.0.1, < 2.0)
     faraday (1.10.0)
@@ -104,13 +108,13 @@ GEM
       webrick
     google-apis-iamcredentials_v1 (0.10.0)
       google-apis-core (>= 0.4, < 2.a)
-    google-apis-storage_v1 (0.11.0)
+    google-apis-storage_v1 (0.13.0)
       google-apis-core (>= 0.4, < 2.a)
     google-cloud-core (1.6.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
-    google-cloud-env (1.5.0)
-      faraday (>= 0.17.3, < 2.0)
+    google-cloud-env (1.6.0)
+      faraday (>= 0.17.3, < 3.0)
     google-cloud-errors (1.2.0)
     google-cloud-storage (1.36.1)
       addressable (~> 2.8)
@@ -128,7 +132,7 @@ GEM
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
     hashdiff (1.0.1)
-    hiera (3.8.0)
+    hiera (3.9.0)
     hocon (1.3.1)
     httpclient (2.8.3)
     i18n (1.10.0)
@@ -143,11 +147,15 @@ GEM
     memoist (0.16.2)
     method_source (1.0.0)
     mini_mime (1.1.2)
+    mini_portile2 (2.8.0)
     minitest (5.15.0)
+    mize (0.4.0)
+      protocol (~> 2.0)
     multi_json (1.15.0)
     multipart-post (2.1.1)
     nio4r (2.5.8)
-    nokogiri (1.13.3-x86_64-darwin)
+    nokogiri (1.13.6)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
     oauth (0.5.8)
     octokit (4.22.0)
@@ -157,6 +165,8 @@ GEM
     parallel (1.21.0)
     parser (3.1.1.0)
       ast (~> 2.4.1)
+    protocol (2.0.0)
+      ruby_parser (~> 3.0)
     protocol-hpack (1.4.2)
     protocol-http (0.22.5)
     protocol-http1 (0.14.2)
@@ -168,7 +178,7 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (4.0.6)
-    puppet (7.14.0)
+    puppet (7.16.0)
       concurrent-ruby (~> 1.0)
       deep_merge (~> 1.0)
       facter (> 2.0.1, < 5)
@@ -224,6 +234,8 @@ GEM
       rubocop (~> 1.19)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
+    ruby_parser (3.19.1)
+      sexp_processor (~> 4.16)
     rubyzip (2.3.2)
     sawyer (0.8.2)
       addressable (>= 2.3.5)
@@ -234,13 +246,17 @@ GEM
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2)
     semantic_puppet (1.0.4)
+    sexp_processor (4.16.1)
     signet (0.16.1)
       addressable (~> 2.8)
       faraday (>= 0.17.5, < 3.0)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
+    sync (0.5.0)
     thor (1.2.1)
     timers (4.3.3)
+    tins (1.31.0)
+      sync
     trailblazer-option (0.1.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)

data/abide_dev_utils.gemspec

@@ -40,6 +40,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'selenium-webdriver', '~> 4.0.0.beta4'
   spec.add_dependency 'google-cloud-storage', '~> 1.34'
   spec.add_dependency 'hashdiff', '~> 1.0'
+  spec.add_dependency 'amatch', '~> 0.4'
 
   # Dev dependencies
   spec.add_development_dependency 'bundler'

data/lib/abide_dev_utils/cem.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'abide_dev_utils/xccdf'
+
+module AbideDevUtils
+  # Methods for working with Compliance Enforcement Modules (CEM)
+  module CEM
+    def self.xccdf
+      return @xccdf if defined?(@xccdf)
+
+      xccdf = Object.new
+      xccdf.extend AbideDevUtils::XCCDF::Common
+      @xccdf = xccdf
+      @xccdf
+    end
+
+    def self.rule_id_format(rule_id)
+      case rule_id
+      when /^c[0-9_]+$/
+        :hiera_title_num
+      when /^[a-z][a-z0-9_]+$/
+        :hiera_title
+      when /^[0-9.]+$/
+        :number
+      else
+        :title
+      end
+    end
+
+    def self.rule_identifiers(rule_id)
+      {
+        number: xccdf.control_parts(rule_id).first,
+        hiera_title: xccdf.name_normalize_control(rule_id),
+        hiera_title_num: xccdf.number_normalize_control(rule_id),
+      }
+    end
+
+    def self.update_legacy_config_from_diff(config_hiera, diff)
+      new_config_hiera = config_hiera.dup
+      new_control_configs = {}
+      change_report = []
+      changes = diff.select { |d| d[:type][0] == :number }
+      config_hiera['config']['control_configs'].each do |key, val_hash|
+        key_id_format = rule_id_format(key)
+        changed = false
+        changes.each do |change|
+          if key_id_format == :title
+            next unless change[:title] == key
+          else
+            next unless rule_identifiers(change[:self].id)[key_id_format] == key
+          end
+
+          changed = true
+          new_key = if key_id_format == :title
+                      change[:other_title]
+                    else
+                      rule_identifiers(change[:other].id)[key_id_format]
+                    end
+          new_control_configs[new_key] = val_hash
+          change_report << {
+            type: :identifier_update,
+            from: key,
+            to: new_key,
+          }
+        end
+        new_control_configs[key] = val_hash unless changed
+      end
+      new_config_hiera['config']['control_configs'] = new_control_configs
+      [new_config_hiera, change_report]
+    end
+  end
+end

data/lib/abide_dev_utils/cli/cem.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'abide_dev_utils/cem'
+require 'abide_dev_utils/files'
+require 'abide_dev_utils/output'
+require 'abide_dev_utils/validate'
+require 'abide_dev_utils/xccdf/diff/benchmark'
+require 'abide_dev_utils/cli/abstract'
+
+module Abide
+  module CLI
+    class CemCommand < AbideCommand
+      CMD_NAME = 'cem'
+      CMD_SHORT = 'Commands related to Puppet CEM'
+      CMD_LONG = 'Namespace for commands related to Puppet CEM'
+      def initialize
+        super(CMD_NAME, CMD_SHORT, CMD_LONG, takes_commands: true)
+        add_command(CemUpdateConfig.new)
+      end
+    end
+
+    class CemUpdateConfig < AbideCommand
+      CMD_NAME = 'update-config'
+      CMD_SHORT = 'Updates the Puppet CEM config'
+      CMD_LONG = 'Updates the Puppet CEM config'
+      def initialize
+        super(CMD_NAME, CMD_SHORT, CMD_LONG, takes_commands: true)
+        add_command(CemUpdateConfigFromDiff.new)
+      end
+    end
+
+    class CemUpdateConfigFromDiff < AbideCommand
+      CMD_NAME = 'from-diff'
+      CMD_SHORT = 'Update by diffing two XCCDF files'
+      CMD_LONG = 'Update by diffing two XCCDF files'
+      CMD_CONFIG_FILE = 'Path to the Puppet CEM config file'
+      CMD_CURRENT_XCCDF = 'Path to the current XCCDF file'
+      CMD_NEW_XCCDF = 'Path to the new XCCDF file'
+      def initialize
+        super(CMD_NAME, CMD_SHORT, CMD_LONG, takes_commands: false)
+        argument_desc(CONFIG_FILE: CMD_CONFIG_FILE, CURRENT_XCCDF: CMD_CURRENT_XCCDF, NEW_XCCDF: CMD_NEW_XCCDF)
+        options.on('-o [FILE]', '--out-file [FILE]', 'Path to save the updated config file') do |o|
+          @data[:out_file] = o
+        end
+        options.on('-v', '--verbose', 'Verbose output') do
+          @data[:verbose] = true
+        end
+        options.on('-q', '--quiet', 'Quiet output') do
+          @data[:quiet] = true
+        end
+      end
+
+      def help_arguments
+        <<~ARGHELP
+          Arguments:
+            CONFIG_FILE:   #{CMD_CONFIG_FILE}
+            CURRENT_XCCDF: #{CMD_CURRENT_XCCDF}
+            NEW_XCCDF:     #{CMD_NEW_XCCDF}
+        ARGHELP
+      end
+
+      def execute(config_file, cur_xccdf, new_xccdf)
+        AbideDevUtils::Validate.file(config_file, extension: 'yaml')
+        AbideDevUtils::Validate.file(cur_xccdf, extension: 'xml')
+        config_hiera = AbideDevUtils::Files::Reader.read(config_file, safe: true)
+        diff = AbideDevUtils::XCCDF::Diff::BenchmarkDiff.new(cur_xccdf, new_xccdf).diff[:diff][:number_title]
+        new_config_hiera, change_report = AbideDevUtils::CEM.update_legacy_config_from_diff(config_hiera, diff)
+        AbideDevUtils::Output.yaml(new_config_hiera, console: @data[:verbose], file: @data[:out_file])
+        AbideDevUtils::Output.simple(change_report) unless @data[:quiet]
+      end
+    end
+  end
+end

data/lib/abide_dev_utils/cli/xccdf.rb

@@ -104,13 +104,24 @@ module Abide
         options.on('-p [PROFILE]', '--profile', 'Only diff and specific profile in the benchmarks') do |x|
           @data[:profile] = x
         end
+        options.on('-l [LEVEL]', '--level', 'Only diff the specific level in the benchmarks') do |x|
+          @data[:level] = x
+        end
+        options.on('-r', '--raw', 'Output the diff in raw hash format') { @data[:raw] = true }
         options.on('-q', '--quiet', 'Show no output in the terminal') { @data[:quiet] = false }
         options.on('--no-diff-profiles', 'Do not diff the profiles in the XCCDF files') { @data[:diff_profiles] = false }
         options.on('--no-diff-controls', 'Do not diff the controls in the XCCDF files') { @data[:diff_controls] = false }
+        options.on('--old-style', 'Use old-style diffs') { @data[:old_style] = true }
       end
 
       def execute(file1, file2)
-        diffreport = AbideDevUtils::XCCDF.diff(file1, file2, @data)
+        diffreport = if @data[:old_style]
+                       AbideDevUtils::XCCDF.diff(file1, file2, @data)
+                     else
+                       dr = AbideDevUtils::XCCDF.new_style_diff(file1, file2, @data)
+                       dr[:diff][:number_title].map! { |d| d[:text] }
+                       dr
+                     end
         AbideDevUtils::Output.yaml(diffreport, console: @data.fetch(:quiet, true), file: @data.fetch(:outfile, nil))
       end
     end

data/lib/abide_dev_utils/cli.rb

@@ -2,6 +2,7 @@
 
 require 'cmdparse'
 require 'abide_dev_utils/version'
+require 'abide_dev_utils/cli/cem'
 require 'abide_dev_utils/constants'
 require 'abide_dev_utils/cli/comply'
 require 'abide_dev_utils/cli/puppet'
@@ -22,6 +23,7 @@ module Abide
       parser.main_options.banner = ROOT_CMD_BANNER
       parser.add_command(CmdParse::HelpCommand.new, default: true)
       parser.add_command(CmdParse::VersionCommand.new(add_switches: true))
+      parser.add_command(CemCommand.new)
       parser.add_command(ComplyCommand.new)
       parser.add_command(PuppetCommand.new)
       parser.add_command(XccdfCommand.new)

data/lib/abide_dev_utils/files.rb

@@ -1,7 +1,41 @@
 # frozen_string_literal: true
 
+require 'abide_dev_utils/validate'
+
 module AbideDevUtils
   module Files
+    class Reader
+      def self.read(path, raw: false, safe: true, opts: {})
+        AbideDevUtils::Validate.file(path)
+        return File.read(path) if raw
+
+        extension = File.extname(path)
+        case extension
+        when /\.yaml|\.yml/
+          require 'yaml'
+          if safe
+            YAML.safe_load(File.read(path))
+          else
+            YAML.load_file(path)
+          end
+        when '.json'
+          require 'json'
+          return JSON.parse(File.read(path), opts) if safe
+
+          JSON.parse!(File.read(path), opts)
+        when '.xml'
+          require 'nokogiri'
+          File.open(path, 'r') do |file|
+            Nokogiri::XML.parse(file) do |config|
+              config.strict.noblanks.norecover
+            end
+          end
+        else
+          File.read(path)
+        end
+      end
+    end
+
     class Writer
       MSG_EXT_APPEND = 'Appending %s extension to file'
 

data/lib/abide_dev_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AbideDevUtils
-  VERSION = "0.9.7"
+  VERSION = "0.10.0"
 end

data/lib/abide_dev_utils/xccdf/diff/benchmark/number_title.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require 'abide_dev_utils/xccdf/diff/benchmark/property_existence'
+
+module AbideDevUtils
+  module XCCDF
+    module Diff
+      # Diffs two XCCDF benchmarks using the title / number of the items as the primary
+      # diff properties.
+      class NumberTitleDiff
+        SKIP_DIFF_TYPES = %i[equal both].freeze
+
+        def initialize(numbered_children, other_numbered_children)
+          new_number_title_objs(numbered_children, other_numbered_children)
+        end
+
+        def diff
+          @diff ||= find_diffs(@number_title_objs, @other_number_title_objs)
+        end
+
+        def to_s
+          parts = []
+          @diff.each do |_, diffs|
+            diffs.each do |dh|
+              parts << dh[:diff_text]
+            end
+          end
+          parts.join("\n")
+        end
+
+        private
+
+        attr_writer :diff
+
+        def added_number_title_objs
+          added_titles = @self_prop_checker.added_titles
+          @other_number_title_objs.select do |nto|
+            added_titles.include?(nto.title)
+          end
+        end
+
+        def removed_number_title_objs
+          removed_titles = @self_prop_checker.removed_titles
+          @number_title_objs.select do |nto|
+            removed_titles.include?(nto.title)
+          end
+        end
+
+        def find_diffs(objs, other_objs)
+          diffs = []
+          added_number_title_objs.each do |nto|
+            change_type = %i[both added]
+            stand_in = NumberTitleContainerStandIn.new(change_type)
+            diffs << process_diffs([diff_hash(change_type, stand_in, nto)])
+          end
+          removed_number_title_objs.each do |nto|
+            change_type = %i[both removed]
+            stand_in = NumberTitleContainerStandIn.new(change_type)
+            diffs << process_diffs([diff_hash(change_type, nto, stand_in)])
+          end
+          objs.each do |obj|
+            obj_diffs = other_objs.each_with_object([]) do |other_obj, o_ary|
+              obj_diff = obj.diff(other_obj)
+              next if SKIP_DIFF_TYPES.include?(obj_diff[0])
+
+              o_ary << diff_hash(obj_diff, obj, other_obj)
+            end
+
+            processed_obj_diffs = process_diffs(obj_diffs)
+            diffs << processed_obj_diffs unless obj_diffs.empty?
+          end
+          diffs
+        end
+
+        def process_diffs(diffs)
+          return {} if diffs.empty?
+
+          raise "Unexpected diffs: #{diffs}" if diffs.length > 2
+
+          return diffs[0] if diffs.length == 1
+
+          if diffs[0][:type][0] == PropChecker.inverse_existence_state[diffs[1][:type][0]]
+            diffs[0]
+          else
+            diffs[1]
+          end
+        end
+
+        def diff_hash(diff_type, obj, other_obj)
+          {
+            self: obj.child,
+            other: other_obj.child,
+            type: diff_type,
+            text: diff_type_text(diff_type, obj, other_obj),
+            number: obj.number,
+            other_number: other_obj.number,
+            title: obj.title,
+            other_title: other_obj.title,
+          }
+        end
+
+        def diff_type_text(diff_type, obj, other_obj)
+          DiffTypeText.text(diff_type, obj, other_obj)
+        end
+
+        def new_number_title_objs(children, other_children)
+          number_title_objs = children.map { |c| NumberTitleContainer.new(c) }.sort
+          other_number_title_objs = other_children.map { |c| NumberTitleContainer.new(c) }.sort
+          @self_prop_checker = PropChecker.new(number_title_objs, other_number_title_objs)
+          @other_prop_checker = PropChecker.new(other_number_title_objs, number_title_objs)
+          number_title_objs.map { |n| n.prop_checker = @self_prop_checker }
+          other_number_title_objs.map { |n| n.prop_checker = @other_prop_checker }
+          @number_title_objs = number_title_objs
+          @other_number_title_objs = other_number_title_objs
+        end
+      end
+
+      # Creates string representations of diff types
+      class DiffTypeText
+        def self.text(diff_type, obj, other_obj)
+          case diff_type[0]
+          when :equal
+            'The objects are equal'
+          when :title
+            "Title changed: Number \"#{obj.number}\": #{obj.title} -> #{other_obj.title}"
+          when :number
+            number_diff_type_text(diff_type, obj, other_obj)
+          when :both
+            both_diff_type_text(diff_type, obj, other_obj)
+          when :add
+            "Add object with number \"#{other_obj.number}\" and title \"#{other_obj.title}\""
+          when :remove
+            "Remove object with number \"#{obj.number}\" and title \"#{obj.title}\""
+          else
+            raise ArgumentError, "Unknown diff type: #{diff_type}"
+          end
+        end
+
+        def self.number_diff_type_text(diff_type, obj, other_obj)
+          case diff_type[1]
+          when :added
+            "Number changed (New Number): Title \"#{obj.title}\": #{obj.number} -> #{other_obj.number}"
+          when :exists
+            "Number changed (Existing Number): Title \"#{obj.title}\": #{obj.number} -> #{other_obj.number}"
+          else
+            raise ArgumentError, "Unknown diff type for number change: #{diff_type[1]}"
+          end
+        end
+
+        def self.both_diff_type_text(diff_type, obj, other_obj)
+          case diff_type[1]
+          when :added
+            "Added object: Title \"#{other_obj.title}\": Number \"#{other_obj.number}\""
+          when :removed
+            "Removed object: Title \"#{obj.title}\": Number \"#{obj.number}\""
+          else
+            raise ArgumentError, "Unknown diff type for both change: #{diff_type[1]}"
+          end
+        end
+      end
+
+      # Checks properties for existence in both benchmarks.
+      class PropChecker < AbideDevUtils::XCCDF::Diff::PropertyExistenceChecker
+        attr_reader :all_numbers, :all_titles, :other_all_numbers, :other_all_titles
+
+        def initialize(number_title_objs, other_number_title_objs)
+          super
+          @all_numbers = number_title_objs.map(&:number)
+          @all_titles = number_title_objs.map(&:title)
+          @other_all_numbers = other_number_title_objs.map(&:number)
+          @other_all_titles = other_number_title_objs.map(&:title)
+        end
+
+        def title(title)
+          property_existence(title, @all_titles, @other_all_titles)
+        end
+
+        def number(number)
+          property_existence(number, @all_numbers, @other_all_numbers)
+        end
+
+        def added_numbers
+          added(@all_numbers, @other_all_numbers)
+        end
+
+        def removed_numbers
+          removed(@all_numbers, @other_all_numbers)
+        end
+
+        def added_titles
+          added(@all_titles, @other_all_titles)
+        end
+
+        def removed_titles
+          removed(@all_titles, @other_all_titles)
+        end
+      end
+
+      class NumberTitleDiffError < StandardError; end
+      class InconsistentDiffTypeError < StandardError; end
+
+      # Holds a number and title for a child of a benchmark
+      # and provides methods to compare it to another child.
+      class NumberTitleContainer
+        include ::Comparable
+        attr_accessor :prop_checker
+        attr_reader :child, :number, :title
+
+        def initialize(child, prop_checker = nil)
+          @child = child
+          @number = child.number.to_s
+          @title = child.title.to_s
+          @prop_checker = prop_checker
+        end
+
+        def diff(other)
+          return %i[equal exist] if number == other.number && title == other.title
+
+          if number == other.number && title != other.title
+            c_diff = correlate_prop_diff_types(@prop_checker.title(other.title),
+                                               other.prop_checker.title(other.title))
+            [:title, c_diff]
+          elsif title == other.title && number != other.number
+            c_diff = correlate_prop_diff_types(@prop_checker.number(other.number),
+                                               other.prop_checker.number(other.number))
+            [:number, c_diff]
+          else
+            %i[both exist]
+          end
+        rescue StandardError => e
+          err_str = [
+            'Error diffing number and title',
+            "Number: #{number}",
+            "Title: #{title}",
+            "Other number: #{other.number}",
+            "Other title: #{other.title}",
+            e.message,
+          ]
+          raise NumberTitleDiffError, err_str.join(', ')
+        end
+
+        def <=>(other)
+          child <=> other.child
+        end
+
+        private
+
+        def correlate_prop_diff_types(self_type, other_type)
+          inverse_diff_type = PropChecker.inverse_existence_state[self_type]
+          return other_type if inverse_diff_type.nil?
+
+          self_type
+        end
+      end
+
+      # Stand-in object for a NumberTitleContainer when the NumberTitleContainer
+      # would not exist. This is used when a change is an add or remove.
+      class NumberTitleContainerStandIn
+        attr_reader :child, :number, :title
+
+        def initialize(change_type)
+          @change_type = change_type
+          @child = nil
+          @number = ''
+          @title = ''
+        end
+      end
+    end
+  end
+end