octocatalog-diff 0.5.1

data/lib/octocatalog-diff/catalog-diff/cli/diffs.rb

@@ -0,0 +1,145 @@
+require_relative '../differ'
+
+module OctocatalogDiff
+  module CatalogDiff
+    class Cli
+      # Wrapper around OctocatalogDiff::CatalogDiff::Differ to provide the logger object, set up
+      # ignores, and add additional ignores for items dependent upon the compilation directory.
+      class Diffs
+        # Constructor
+        # @param options [Hash] Options from cli/options
+        # @param logger [Logger] Logger object
+        def initialize(options, logger)
+          @options = options
+          @logger = logger
+        end
+
+        # The method to call externally, passing in the catalogs as a hash (see parameter). This
+        # sets up options and ignores and then actually performs the diffs. The result is the array
+        # of diffs.
+        # @param catalogs [Hash] { :to => OctocatalogDiff::Catalog, :from => OctocatalogDiff::Catalog }
+        # @return [Array<diffs>] Array of diffs
+        def diffs(catalogs)
+          @logger.debug 'Begin compute diffs between catalogs'
+          diff_opts = @options.merge(logger: @logger)
+
+          # Construct the actual differ object that the present one wraps
+          differ = OctocatalogDiff::CatalogDiff::Differ.new(diff_opts, catalogs[:from], catalogs[:to])
+          differ.ignore(attr: 'tags') unless @options.fetch(:include_tags, false)
+          differ.ignore(@options.fetch(:ignore, []))
+
+          # Handle --ignore-tags option, the ability to tag resources within modules/manifests and
+          # have catalog-diff ignore them.
+          if @options[:ignore_tags].is_a?(Array) && @options[:ignore_tags].any?
+            # Go through the "to" catalog and identify any resources that have been tagged with one or more
+            # specified "ignore tags." Add any such items to the ignore list. The 'to' catalog has the authoritative
+            # list of dynamic ignores.
+            catalogs[:to].resources.each do |resource|
+              next unless tagged_for_ignore?(resource)
+              differ.ignore(type: resource['type'], title: resource['title'])
+              @logger.debug "Ignoring type='#{resource['type']}', title='#{resource['title']}' based on tag in to-catalog"
+            end
+
+            # Go through the "from" catalog and identify any resources that have been tagged with one or more
+            # specified "ignore tags." Only mark the resources for ignoring if they do not appear in the 'to'
+            # catalog, thereby allowing the 'to' catalog to be the authoritative ignore list. This allows deleted
+            # items that were previously ignored to continue to be ignored.
+            catalogs[:from].resources.each do |resource|
+              next if catalogs[:to].resource(type: resource['type'], title: resource['title'])
+              next unless tagged_for_ignore?(resource)
+              differ.ignore(type: resource['type'], title: resource['title'])
+              @logger.debug "Ignoring type='#{resource['type']}', title='#{resource['title']}' based on tag in from-catalog"
+            end
+          end
+
+          # Actually perform the diff
+          diff_result = differ.diff
+          diff_result.delete_if { |element| change_due_to_compilation_dir?(element, catalogs) }
+          @logger.debug 'Success compute diffs between catalogs'
+          diff_result
+        end
+
+        # Catch anything that explictly changed as a result of different compilation directories and
+        # warn about it. These are probably things that should be refactored. For now we're going to pull
+        # these out after the fact so we can warn about them if they do show up.
+        # @param change [Array(diff format)] Change in diff format
+        # @param catalogs [Hash] { :to => OctocatalogDiff::Catalog, :from => OctocatalogDiff::Catalog }
+        # @return [Boolean] True if change includes compilation directory, false otherwise
+        def change_due_to_compilation_dir?(change, catalogs)
+          dir1 = catalogs.fetch(:to).compilation_dir
+          dir2 = catalogs.fetch(:from).compilation_dir
+          return false if dir1.nil? || dir2.nil?
+
+          dir1_rexp = Regexp.escape(dir1)
+          dir2_rexp = Regexp.escape(dir2)
+          dir = "(?:#{dir1_rexp}|#{dir2_rexp})"
+
+          # Check for added/removed resources where the title of the resource includes the compilation directory
+          if change[0] == '+' || change[0] == '-'
+            if change[1] =~ /^([^\f]+)\f([^\f]*#{dir}[^\f]*)/
+              message = "Resource #{Regexp.last_match(1)}[#{Regexp.last_match(2)}]"
+              message += ' appears to depend on catalog compilation directory. Suppressed from results.'
+              @logger.warn message
+              return true
+            end
+          end
+
+          # Check for a change where the difference in a parameter exactly corresponds to the difference in the
+          # compilation directory.
+          if change[0] == '~' || change[0] == '!'
+            from_before = nil
+            from_after = nil
+            from_match = false
+            to_before = nil
+            to_after = nil
+            to_match = false
+
+            if change[2] =~ /^(.*)#{dir2}(.*)$/m
+              from_before = Regexp.last_match(1) || ''
+              from_after = Regexp.last_match(2) || ''
+              from_match = true
+            end
+
+            if change[3] =~ /^(.*)#{dir1}(.*)$/m
+              to_before = Regexp.last_match(1) || ''
+              to_after = Regexp.last_match(2) || ''
+              to_match = true
+            end
+
+            if from_match && to_match && to_before == from_before && to_after == from_after
+              message = "Resource key #{change[1].gsub(/\f/, ' => ')}"
+              message += ' appears to depend on catalog compilation directory. Suppressed from results.'
+              @logger.warn message
+              return true
+            end
+
+            if from_match || to_match
+              message = "Resource key #{change[1].gsub(/\f/, ' => ')}"
+              message += ' may depend on catalog compilation directory, but there may be differences.'
+              message += ' This is included in results for now, but please verify.'
+              @logger.warn message
+            end
+          end
+
+          false
+        end
+
+        private
+
+        # Determine if a resource is tagged with any ignore-tag.
+        # @param resource [Hash] The resource
+        # @return [Boolean] true if tagged for ignore, false if not
+        def tagged_for_ignore?(resource)
+          return false unless @options[:ignore_tags].is_a?(Array)
+          return false unless resource.key?('tags') && resource['tags'].is_a?(Array)
+          @options[:ignore_tags].each do |tag|
+            # tag_with_type will be like: 'ignored_catalog_diff__mymodule__mytype'
+            tag_with_type = [tag, resource['type'].downcase.gsub(/\W/, '_')].join('__')
+            return true if resource['tags'].include?(tag) || resource['tags'].include?(tag_with_type)
+          end
+          false
+        end
+      end
+    end
+  end
+end

data/lib/octocatalog-diff/catalog-diff/cli/helpers/fact_override.rb

@@ -0,0 +1,99 @@
+require 'json'
+
+module OctocatalogDiff
+  module CatalogDiff
+    class Cli
+      class Helpers
+        # Since input from the command line is in the format of a string, this class helps to guess
+        # at the data type.
+        class FactOverride
+          # Accessors
+          attr_reader :key, :value
+
+          # Constructor: Input will be a string (since it comes from command line).
+          # This code will make a best guess at the data type (or use a supplied data type if any).
+          # @param input [String] Input in the format: key=(data type)value
+          def initialize(input, key = nil)
+            # Normally the input will be a string in the format key=(data type)value where the data
+            # type is optional and the parentheses are literal. Example:
+            #   foo=1            (auto-determine data type - in this case it would be a fixnum)
+            #   foo=(fixnum)1    (will be a fixnum)
+            #   foo=(string)1    (will be '1' the string)
+            # If input is not a string, we can still construct the object if the key is given.
+            # That input would come directly from code and not from the command line, since inputs
+            # from the command line are always strings.
+            if input.is_a?(String)
+              unless input.include?('=')
+                raise ArgumentError, "Fact override '#{input}' is not in 'key=(data type)value' format"
+              end
+              input.strip!
+              @key, raw_value = input.split('=', 2)
+              @value = parsed_value(raw_value)
+            elsif key.nil?
+              message = "Define a key when the input is not a string (#{input.class} => #{input.inspect})"
+              raise ArgumentError, message
+            else
+              @key = key
+              @value = input
+            end
+          end
+
+          private
+
+          # Guess the datatype from a particular input
+          # @param input [String] Input in string format
+          # @return [?] Output in appropriate format
+          def parsed_value(input)
+            # If data type is explicitly given
+            if input =~ /^\((\w+)\)(.*)$/m
+              datatype = Regexp.last_match(1)
+              value = Regexp.last_match(2)
+              return convert_to_data_type(datatype.downcase, value)
+            end
+
+            # Guess data type
+            return input.to_i if input =~ /^-?\d+$/
+            return input.to_f if input =~ /^-?\d*\.\d+$/
+            return true if input.casecmp('true').zero?
+            return false if input.casecmp('false').zero?
+            input
+          end
+
+          # Handle data type that's explicitly given
+          # @param datatype [String] Data type (as a string)
+          # @param value [String] Value given
+          # @return [?] Value converted to specified data type
+          def convert_to_data_type(datatype, value)
+            return value if datatype == 'string'
+            return parse_json(value) if datatype == 'json'
+            return nil if datatype == 'nil'
+            if datatype == 'fixnum'
+              return Regexp.last_match(1).to_i if value =~ /^(-?\d+)$/
+              raise ArgumentError, "Illegal fixnum '#{value}'"
+            end
+            if datatype == 'float'
+              return Regexp.last_match(1).to_f if value =~ /^(-?\d*\.\d+)$/
+              return Regexp.last_match(1).to_f if value =~ /^(-?\d+)$/
+              raise ArgumentError, "Illegal float '#{value}'"
+            end
+            if datatype == 'boolean'
+              return true if value.casecmp('true').zero?
+              return false if value.casecmp('false').zero?
+              raise ArgumentError, "Illegal boolean '#{value}'"
+            end
+            raise ArgumentError, "Unknown data type '#{datatype}'"
+          end
+
+          # Parse JSON value
+          # @param input [String] Input, hopefully in JSON format
+          # @return [?] Output data structure
+          def parse_json(input)
+            JSON.parse(input)
+          rescue JSON::ParserError => exc
+            raise JSON::ParserError, "Failed to parse JSON: input=#{input} error=#{exc}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/octocatalog-diff/catalog-diff/cli/options.rb

@@ -0,0 +1,173 @@
+require_relative '../cli'
+require_relative '../../facts'
+require_relative '../../version'
+
+require 'optparse'
+
+module OctocatalogDiff
+  module CatalogDiff
+    class Cli
+      # This class contains the option parser. 'parse_options' is the external entry point.
+      class Options
+        # The usage banner.
+        BANNER = 'Usage: catalog-diff -n <hostname> [-f <from environment>] [-t <to environment>]'.freeze
+
+        # List of classes
+        def self.classes
+          @classes ||= []
+        end
+
+        # Define the Option class and newoption() method for use by catalog-diff/cli/options/*.rb files
+        class Option
+          DEFAULT_WEIGHT = 999
+          def self.has_weight(w) # rubocop:disable Style/PredicateName
+            @weight = w
+          end
+
+          def self.weight
+            @weight || DEFAULT_WEIGHT
+          end
+
+          def self.newoption(name, &block)
+            klass = Class.new(OctocatalogDiff::CatalogDiff::Cli::Options::Option)
+            klass.const_set('NAME', name)
+            klass.class_exec(&block)
+            Options.classes.push(klass)
+          end
+        end
+
+        # Method to call all of the other methods in this class. Except in very specific circumstances,
+        # this should be the method called from outside of this class.
+        # @param argv [Array] Array of command line arguments
+        # @param defaults [Hash] Default values
+        # @return [Hash] Parsed options
+        def self.parse_options(argv, defaults = {})
+          options = defaults.dup
+          Options.classes.clear
+          ::OptionParser.new do |parser|
+            parser.banner = "#{BANNER}\n\n"
+            option_classes.each do |klass|
+              obj = klass.new
+              obj.parse(parser, options)
+            end
+            parser.on_tail('-v', '--version', 'Show version information about this program and quit.') do
+              puts "octocatalog-diff #{OctocatalogDiff::Version::VERSION}"
+              exit
+            end
+          end.parse! argv
+          options
+        end
+
+        # Read in *.rb files in the 'options' directory and create classes from them.
+        # Sort the classes according to weight and return the list of sorted classes.
+        # @return [Array<Class>] Sorted classes
+        def self.option_classes
+          files = Dir.glob(File.join(File.dirname(__FILE__), 'options', '*.rb'))
+          files.each { |file| load file } # Populates self.classes
+          classes.sort { |a, b| a.weight <=> b.weight }
+        end
+
+        # Sets up options that can be defined globally or for just one branch. For example, with a
+        # CLI name of 'puppet-binary' this will acknowledge 3 options: --puppet-binary (global),
+        # --from-puppet-binary (for the from branch only), and --to-puppet-binary (for the to branch
+        # only). The only options that will be created are the 'to' and 'from' variants, but the global
+        # option will populate any of the 'to' and 'from' variants that are missing.
+        # @param :datatype [?] Expected data type
+        def self.option_globally_or_per_branch(opts = {})
+          datatype = opts.fetch(:datatype, '')
+          return option_globally_or_per_branch_string(opts) if datatype.is_a?(String)
+          return option_globally_or_per_branch_array(opts) if datatype.is_a?(Array)
+          raise ArgumentError, "option_globally_or_per_branch not equipped to handle #{datatype.class}"
+        end
+
+        # See description of `option_globally_or_per_branch`. This implements the logic for a string value.
+        # @param :parser [OptionParser object] The OptionParser argument
+        # @param :options [Hash] Options hash being constructed; this is modified in this method.
+        # @param :cli_name [String] Name of option on command line (e.g. puppet-binary)
+        # @param :option_name [Symbol] Name of option in the options hash (e.g. :puppet_binary)
+        # @param :desc [String] Description of option on the command line; will have "for the XX branch" appended
+        def self.option_globally_or_per_branch_string(opts)
+          parser = opts.fetch(:parser)
+          options = opts.fetch(:options)
+          cli_name = opts.fetch(:cli_name)
+          option_name = opts.fetch(:option_name)
+          desc = opts.fetch(:desc)
+
+          flag = "#{cli_name} STRING"
+          from_option = "from_#{option_name}".to_sym
+          to_option = "to_#{option_name}".to_sym
+          parser.on("--#{flag}", "#{desc} globally") do |x|
+            validate_option(opts[:validator], x) if opts[:validator]
+            translated = translate_option(opts[:translator], x)
+            options[to_option] ||= translated
+            options[from_option] ||= translated
+          end
+          parser.on("--to-#{flag}", "#{desc} for the to branch") do |x|
+            validate_option(opts[:validator], x) if opts[:validator]
+            options[to_option] = translate_option(opts[:translator], x)
+          end
+          parser.on("--from-#{flag}", "#{desc} for the from branch") do |x|
+            validate_option(opts[:validator], x) if opts[:validator]
+            options[from_option] = translate_option(opts[:translator], x)
+          end
+        end
+
+        # See description of `option_globally_or_per_branch`. This implements the logic for an array.
+        # @param :parser [OptionParser object] The OptionParser argument
+        # @param :options [Hash] Options hash being constructed; this is modified in this method.
+        # @param :cli_name [String] Name of option on command line (e.g. puppet-binary)
+        # @param :option_name [Symbol] Name of option in the options hash (e.g. :puppet_binary)
+        # @param :desc [String] Description of option on the command line; will have "for the XX branch" appended
+        def self.option_globally_or_per_branch_array(opts = {})
+          parser = opts.fetch(:parser)
+          options = opts.fetch(:options)
+          cli_name = opts.fetch(:cli_name)
+          option_name = opts.fetch(:option_name)
+          desc = opts.fetch(:desc)
+
+          flag = "#{cli_name} STRING1[,STRING2[,...]]"
+          from_option = "from_#{option_name}".to_sym
+          to_option = "to_#{option_name}".to_sym
+          parser.on("--#{flag}", Array, "#{desc} globally") do |x|
+            validate_option(opts[:validator], x) if opts[:validator]
+            translated = translate_option(opts[:translator], x)
+            options[to_option] ||= []
+            options[to_option].concat translated
+            options[from_option] ||= []
+            options[from_option].concat translated
+          end
+          parser.on("--to-#{flag}", Array, "#{desc} for the to branch") do |x|
+            validate_option(opts[:validator], x) if opts[:validator]
+            options[to_option] ||= []
+            options[to_option].concat translate_option(opts[:translator], x)
+          end
+          parser.on("--from-#{flag}", Array, "#{desc} for the from branch") do |x|
+            validate_option(opts[:validator], x) if opts[:validator]
+            options[from_option] ||= []
+            options[from_option].concat translate_option(opts[:translator], x)
+          end
+        end
+
+        # If a validator was provided, run the validator on the supplied value. The validator is expected to
+        # throw an error if there is a problem. Note that the validator runs *before* the translator if both
+        # a validator and translator are supplied.
+        # @param validator [Code] Validation function
+        # @param value [?] Value to validate (typically a String but can really be anything)
+        def self.validate_option(validator, value)
+          validator.call(value)
+        end
+
+        # If a translator was provided, run the translator on the supplied value. The translator is expected
+        # to return the data type needed for the option (typically a String but can really be anything). Note
+        # that the translator runs *after* the validator if both a validator and translator are supplied.
+        # @param translator [Code] Translator function
+        # @param value [?] Original input value
+        # @return [?] Translated value
+        def self.translate_option(translator, value)
+          return value if translator.nil?
+          translator.call(value)
+        end
+      end
+    end
+  end
+end

data/lib/octocatalog-diff/catalog-diff/cli/options/basedir.rb

@@ -0,0 +1,14 @@
+# Option to set the base checkout directory of puppet repository
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::CatalogDiff::Cli::Options::Option.newoption(:basedir) do
+  has_weight 10
+
+  def parse(parser, options)
+    parser.on('--basedir DIRNAME', 'Use an alternate base directory (git checkout of puppet repository)') do |dir|
+      path = File.absolute_path(dir)
+      raise Errno::ENOENT, 'Invalid basedir provided' unless Dir.exist?(path)
+      options[:basedir] = path
+    end
+  end
+end

data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_environment.rb

@@ -0,0 +1,18 @@
+# Allow the bootstrap environment to be set up via the command line.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::CatalogDiff::Cli::Options::Option.newoption(:bootstrap_environment) do
+  has_weight 50
+
+  def parse(parser, options)
+    descriptive_text = 'Bootstrap script environment variables in key=value format'
+    parser.on('--bootstrap-environment "key1=val1,key2=val2,..."', Array, descriptive_text) do |res|
+      options[:bootstrap_environment] ||= {}
+      res.each do |item|
+        raise ArgumentError, "Bootstrap environment #{item} must be in key=value format!" unless item =~ /=/
+        key, val = item.split(/=/, 2)
+        options[:bootstrap_environment][key] = Regexp.last_match(1) if val.strip =~ /^['"]?(.+?)['"]?$/
+      end
+    end
+  end
+end