octocatalog-diff 1.5.2 → 2.1.0

data/doc/requirements.md

@@ -2,10 +2,14 @@
 
 To run `octocatalog-diff` you will need these basics:
 
-- Ruby 2.0 through 2.4 (we test octocatalog-diff with Ruby 2.0, 2.1, 2.2, 2.3, and 2.4)
+- An appropriate Puppet version and [corresponding ruby version](https://puppet.com/docs/puppet/5.4/system_requirements.html)
+  - Puppet 5.x officially supports Ruby 2.4
+  - Puppet 4.x officially supports Ruby 2.1, but seems to work fine with later versions as well
+  - Puppet 3.8.7 -- we attempt to maintain compatibility in `octocatalog-diff` to facilitate upgrades even though this version is no longer supported by Puppet
+  - We don't officially support Puppet 3.8.6 or before
 - Mac OS, Linux, or other Unix-line operating system (Windows is not supported)
 - Ability to install gems, e.g. with [rbenv](https://github.com/rbenv/rbenv) or [rvm](https://rvm.io/), or root privileges to install into the system Ruby
-- Puppet agent for [Linux](https://docs.puppet.com/puppet/latest/reference/install_linux.html) or [Mac OS X](https://docs.puppet.com/puppet/latest/reference/install_osx.html), or installed as a gem (we support Puppet 3.8.7 and all versions of Puppet 4.x)
+- Puppet agent for [Linux](https://docs.puppet.com/puppet/latest/reference/install_linux.html) or [Mac OS X](https://docs.puppet.com/puppet/latest/reference/install_osx.html), or installed as a gem
 
 We recommend that you also have the following to get the most out of `octocatalog-diff`, but these are not absolute requirements:
 

data/lib/octocatalog-diff/catalog-diff/differ.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'diffy'
+require 'digest'
 require 'hashdiff'
 require 'json'
 require 'set'
@@ -11,6 +12,8 @@ require_relative '../errors'
 require_relative '../util/util'
 require_relative 'filter'
 
+HashDiff = Hashdiff unless defined? HashDiff
+
 module OctocatalogDiff
   module CatalogDiff
     # Calculate the difference between two Puppet catalogs.
@@ -263,7 +266,7 @@ module OctocatalogDiff
 
             # Handle parameters
             if k == 'parameters'
-              cleansed_param = cleanse_parameters_hash(v)
+              cleansed_param = cleanse_parameters_hash(v, resource.fetch('sensitive_parameters', []))
               hsh[k] = cleansed_param unless cleansed_param.nil? || cleansed_param.empty?
             elsif k == 'tags'
               # The order of tags is unimportant. Sort this array to avoid false diffs if order changes.
@@ -294,10 +297,13 @@ module OctocatalogDiff
         # Use diffy to get only the lines that have changed in a text object.
         # As we iterate through the diff, jump out if we have our answer: either
         # true if '=~>' finds ANY match, or false if '=&>' fails to find a match.
-        Diffy::Diff.new(old_val, new_val, context: 0).each do |line|
+        diffy_result = Diffy::Diff.new(old_val, new_val, context: 0)
+        newline_alerts = diffy_result.count { |line| line.strip == '\' }
+        diffy_result.each do |line|
           if regex.match(line.strip)
             return true if operator == '=~>'
           elsif operator == '=&>'
+            next if line.strip == '\' && newline_alerts == 2
             return false
           end
         end
@@ -334,7 +340,8 @@ module OctocatalogDiff
         #   =->  Attribute must have been removed and equal this
         #   =~>  Change must match regexp (one line of change matching is sufficient)
         #   =&>  Change must match regexp (all lines of change MUST match regexp)
-        if rule_attr =~ /\A(.+?)(=[\-\+~&]?>)(.+)/m
+        #   =s>  Change must be array and contain identical elements, ignoring order
+        if rule_attr =~ /\A(.+?)(=[\-\+~&s]?>)(.+)/m
           rule_attr = Regexp.last_match(1)
           operator = Regexp.last_match(2)
           value = Regexp.last_match(3)
@@ -355,6 +362,9 @@ module OctocatalogDiff
               raise RegexpError, "Invalid ignore regexp for #{key}: #{exc.message}"
             end
             matcher = ->(x, y) { regexp_operator_match?(operator, my_regex, x, y) }
+          elsif operator == '=s>'
+            raise ArgumentError, "Invalid ignore option for =s>, must be '='" unless value == '='
+            matcher = ->(x, y) { x.is_a?(Array) && y.is_a?(Array) && Set.new(x) == Set.new(y) }
           end
         end
 
@@ -394,6 +404,13 @@ module OctocatalogDiff
           return false unless rule[:title].casecmp(hsh[:title]).zero?
         end
 
+        # If rule[:attr] is a regular expression, handle that case here.
+        if rule[:attr].is_a?(Regexp)
+          return false unless hsh[:attr].is_a?(String)
+          return false unless rule[:attr].match(hsh[:attr])
+          return ignore_match_true(hsh, rule)
+        end
+
         # Special 'attributes': Ignore specific diff types (+ add, - remove, ~ and ! change)
         if rule[:attr] =~ /\A[\-\+~!]+\Z/
           return ignore_match_true(hsh, rule) if rule[:attr].include?(diff_type)
@@ -446,10 +463,18 @@ module OctocatalogDiff
 
       # Cleanse parameters of filtered attributes.
       # @param parameters_hash [Hash] Hash of parameters
+      # @param sensitive_parameters [Array] Array of sensitive parameters
       # @return [Hash] Cleaned parameters hash (original input hash is not altered)
-      def cleanse_parameters_hash(parameters_hash)
+      def cleanse_parameters_hash(parameters_hash, sensitive_parameters)
         result = parameters_hash.dup
 
+        # hides sensitive params. We still need to know if there's a going to
+        # be a diff, so we hash the value.
+        sensitive_parameters.each do |p|
+          md5 = Digest::MD5.hexdigest Marshal.dump(result[p])
+          result[p] = 'Sensitive [md5sum ' + md5 + ']'
+        end
+
         # 'before' and 'require' handle internal Puppet ordering but do not affect what
         # happens on the target machine. Don't consider these for the purpose of catalog diff.
         result.delete('before')
@@ -510,9 +535,11 @@ module OctocatalogDiff
         catalog2_resources = catalog2_in[:catalog]
 
         @logger.debug "Entering hashdiff_initial; catalog sizes: #{catalog1_resources.size}, #{catalog2_resources.size}"
+        use_lcs = @opts.fetch(:use_lcs, true)
+        @logger.debug "HashDiff configuration: (use_lcs: #{use_lcs})"
         result = []
         hashdiff_add_remove = Set.new
-        hashdiff_result = HashDiff.diff(catalog1_resources, catalog2_resources, delimiter: "\f")
+        hashdiff_result = HashDiff.diff(catalog1_resources, catalog2_resources, delimiter: "\f", use_lcs: use_lcs)
         hashdiff_result.each do |obj|
           # Regular change
           if obj[0] == '~'

data/lib/octocatalog-diff/catalog-diff/filter/compilation_dir.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../filter'
+require_relative '../../util/util'
 
 module OctocatalogDiff
   module CatalogDiff
@@ -35,43 +36,46 @@ module OctocatalogDiff
 
           # Check for a change where the difference in a parameter exactly corresponds to the difference in the
           # compilation directory.
-          if diff.change? && (diff.old_value.is_a?(String) || diff.new_value.is_a?(String))
-            from_before = nil
-            from_after = nil
-            from_match = false
-            to_before = nil
-            to_after = nil
-            to_match = false
+          if diff.change?
+            o = remove_compilation_dir(diff.old_value, dir2)
+            n = remove_compilation_dir(diff.new_value, dir1)
 
-            if diff.old_value =~ /^(.*)#{dir2}(.*)$/m
-              from_before = Regexp.last_match(1) || ''
-              from_after = Regexp.last_match(2) || ''
-              from_match = true
-            end
-
-            if diff.new_value =~ /^(.*)#{dir1}(.*)$/m
-              to_before = Regexp.last_match(1) || ''
-              to_after = Regexp.last_match(2) || ''
-              to_match = true
-            end
-
-            if from_match && to_match && to_before == from_before && to_after == from_after
+            if o != diff.old_value || n != diff.new_value
               message = "Resource key #{diff.type}[#{diff.title}] #{diff.structure.join(' => ')}"
-              message += ' appears to depend on catalog compilation directory. Suppressed from results.'
+              message += ' may depend on catalog compilation directory, but there may be differences.'
+              message += ' This is included in results for now, but please verify.'
               @logger.warn message
-              return true
             end
 
-            if from_match || to_match
+            if o == n
               message = "Resource key #{diff.type}[#{diff.title}] #{diff.structure.join(' => ')}"
-              message += ' may depend on catalog compilation directory, but there may be differences.'
-              message += ' This is included in results for now, but please verify.'
+              message += ' appears to depend on catalog compilation directory. Suppressed from results.'
               @logger.warn message
+              return true
             end
           end
 
           false
         end
+
+        def remove_compilation_dir(v, dir)
+          value = OctocatalogDiff::Util::Util.deep_dup(v)
+          traverse(value) do |e|
+            e.gsub!(dir, '') if e.respond_to?(:gsub!)
+          end
+          value
+        end
+
+        def traverse(a)
+          case a
+          when Array
+            a.map { |v| traverse(v, &Proc.new) }
+          when Hash
+            traverse(a.values, &Proc.new)
+          else
+            yield a
+          end
+        end
       end
     end
   end

data/lib/octocatalog-diff/catalog-util/command.rb

@@ -54,9 +54,21 @@ module OctocatalogDiff
         raise ArgumentError, 'Puppet binary was not supplied' if @puppet_binary.nil?
         raise Errno::ENOENT, "Puppet binary #{@puppet_binary} doesn't exist" unless File.file?(@puppet_binary)
 
+        puppet_version = Gem::Version.new(@options[:puppet_version])
+
         # Node to compile
         cmdline = []
-        cmdline.concat ['master', '--compile', Shellwords.escape(@node)]
+        # The 'puppet master --compile' command was removed in Puppet 6.x and replaced in
+        # Puppet 6.5 with an identically functioning 'puppet catalog compile' command.
+        # From versions 6.0.0 until 6.5.0 there is no compatible invocation method.
+        if puppet_version < Gem::Version.new('6.0.0')
+          cmdline.concat ['master', '--compile', Shellwords.escape(@node)]
+        elsif puppet_version < Gem::Version.new('6.5.0')
+          raise OctocatalogDiff::Errors::PuppetVersionError,
+                'Octocatalog-diff does not support Puppet versions >= 6.0.0 and < 6.5.0'
+        else
+          cmdline.concat ['catalog', 'compile', Shellwords.escape(@node)]
+        end
 
         # storeconfigs?
         if @options[:storeconfigs]
@@ -93,11 +105,21 @@ module OctocatalogDiff
         # Some typical options for puppet
         cmdline.concat %w(
           --no-daemonize
-          --no-ca
           --color=false
-          --config_version="/bin/echo catalogscript"
         )
 
+        if puppet_version < Gem::Version.new('6.0.0')
+          # This config_version parameter causes an error when run with Puppet 6.x. Per
+          # the Puppet configuration settings docs, the below config_version argument
+          # may not actually be valid, but for backward compatibility's sake we'll keep it
+          # for the versions it has always worked with:
+          cmdline.concat ['--config_version="/bin/echo catalogscript"']
+
+          # The 'ca' configuration option was removed in Puppet 6, but we'll keep it
+          # for older versions:
+          cmdline.concat ['--no-ca']
+        end
+
         # Add environment - only make this variable if preserve_environments is used.
         # If preserve_environments is not used, the hard-coded 'production' here matches
         # up with the symlink created under the temporary directory structure.

data/lib/octocatalog-diff/catalog-util/fileresources.rb

@@ -13,9 +13,16 @@ module OctocatalogDiff
       # Public method: Convert file resources to text. See the description of the class
       # just above for details.
       # @param obj [OctocatalogDiff::Catalog] Catalog object (will be modified)
+      # @param environment [String] Environment (defaults to production)
       def self.convert_file_resources(obj, environment = 'production')
         return unless obj.valid? && obj.compilation_dir.is_a?(String) && !obj.compilation_dir.empty?
-        _convert_file_resources(obj.resources, obj.compilation_dir, environment)
+        _convert_file_resources(
+          obj.resources,
+          obj.compilation_dir,
+          environment,
+          obj.options[:compare_file_text_ignore_tags],
+          obj.options[:tag]
+        )
         begin
           obj.catalog_json = ::JSON.generate(obj.catalog)
         rescue ::JSON::GeneratorError => exc
@@ -59,7 +66,7 @@ module OctocatalogDiff
           result = []
           Regexp.last_match(1).split(/:/).map(&:strip).each do |path|
             next if path.start_with?('$')
-            result << File.expand_path(path, dir)
+            result.concat(Dir.glob(File.expand_path(path, dir)))
           end
           result
         else
@@ -70,8 +77,11 @@ module OctocatalogDiff
       # Internal method: Static method to convert file resources. The compilation directory is
       # required, or else this is a no-op. The passed-in array of resources is modified by this method.
       # @param resources [Array<Hash>] Array of catalog resources
-      # @param compilation_dir [String] Compilation directory (so files can be looked up)
-      def self._convert_file_resources(resources, compilation_dir, environment = 'production')
+      # @param compilation_dir [String] Compilation directory
+      # @param environment [String] Environment
+      # @param ignore_tags [Array<String>] Tags that exempt a resource from conversion
+      # @param to_from_tag [String] Either "to" or "from" for catalog type
+      def self._convert_file_resources(resources, compilation_dir, environment, ignore_tags, to_from_tag)
         # Calculate compilation directory. There is not explicit error checking here because
         # there is on-demand, explicit error checking for each file within the modification loop.
         return unless compilation_dir.is_a?(String) && compilation_dir != ''
@@ -88,14 +98,10 @@ module OctocatalogDiff
         resources.map! do |resource|
           if resource_convertible?(resource)
             path = file_path(resource['parameters']['source'], modulepaths)
-            if path.nil?
-              # Pass this through as a wrapped exception, because it's more likely to be something wrong
-              # in the catalog itself than it is to be a broken setup of octocatalog-diff.
-              message = "Errno::ENOENT: Unable to resolve '#{resource['parameters']['source']}'!"
-              raise OctocatalogDiff::Errors::CatalogError, message
-            end
 
-            if File.file?(path)
+            if resource['tags'] && ignore_tags && (resource['tags'] & ignore_tags).any?
+              # Resource tagged not to be converted -- do nothing.
+            elsif path && File.file?(path)
               # If the file is found, read its content. If the content is all ASCII, substitute it into
               # the 'content' parameter for easier comparison. If not, instead populate the md5sum.
               # Delete the 'source' attribute as well.
@@ -103,23 +109,40 @@ module OctocatalogDiff
               is_ascii = content.force_encoding('UTF-8').ascii_only?
               resource['parameters']['content'] = is_ascii ? content : '{md5}' + Digest::MD5.hexdigest(content)
               resource['parameters'].delete('source')
-            elsif File.exist?(path)
+            elsif path && File.exist?(path)
               # We are not handling recursive file installs from a directory or anything else.
               # However, the fact that we found *something* at this location indicates that the catalog
               # is probably correct. Hence, the very general .exist? check.
+            elsif to_from_tag == 'from'
+              # Don't raise an exception for an invalid source in the "from"
+              # catalog, because the developer may be fixing this in the "to"
+              # catalog. If it's broken in the "to" catalog as well, the
+              # exception will be raised when this code runs on that catalog.
             else
-              # This is probably a bug
-              # :nocov:
-              raise "Unable to find '#{resource['parameters']['source']}' at #{path}!"
-              # :nocov:
+              # Pass this through as a wrapped exception, because it's more likely to be something wrong
+              # in the catalog itself than it is to be a broken setup of octocatalog-diff.
+              #
+              # Example error: <OctocatalogDiff::Errors::CatalogError: Unable to resolve
+              # source=>'puppet:///modules/test/tmp/bar' in File[/tmp/bar]
+              # (/x/modules/test/manifests/init.pp:46)>
+              source = resource['parameters']['source']
+              type = resource['type']
+              title = resource['title']
+              file = resource['file'].sub(Regexp.new('^' + Regexp.escape(env_dir) + '/'), '')
+              line = resource['line']
+              message = "Unable to resolve source=>'#{source}' in #{type}[#{title}] (#{file}:#{line})"
+              raise OctocatalogDiff::Errors::CatalogError, message
             end
           end
+
           resource
         end
       end
 
       # Internal method: Determine if a resource is convertible. It is convertible if it
       # is a file resource with no declared 'content' and with a declared and parseable 'source'.
+      # It is not convertible if the resource is tagged with one of the tags declared by
+      # the option `--compare-file-text-ignore-tags`.
       # @param resource [Hash] Resource to check
       # @return [Boolean] True of resource is convertible, false if not
       def self.resource_convertible?(resource)

data/lib/octocatalog-diff/catalog.rb

@@ -191,6 +191,8 @@ module OctocatalogDiff
       build
       raise OctocatalogDiff::Errors::CatalogError, 'Catalog does not appear to have been built' if !valid? && error_message.nil?
       raise OctocatalogDiff::Errors::CatalogError, error_message unless valid?
+      # Handle the structure returned by the /puppet/v4/catalog Puppetserver endpoint:
+      return @catalog['catalog']['resources'] if @catalog['catalog'].is_a?(Hash) && @catalog['catalog']['resources'].is_a?(Array)
       return @catalog['data']['resources'] if @catalog['data'].is_a?(Hash) && @catalog['data']['resources'].is_a?(Array)
       return @catalog['resources'] if @catalog['resources'].is_a?(Array)
       # This is a bug condition
@@ -304,20 +306,36 @@ module OctocatalogDiff
         unless res =~ /\A([\w:]+)\[(.+)\]\z/
           raise ArgumentError, "Resource #{res} is not in the expected format"
         end
-        resource(type: Regexp.last_match(1), title: Regexp.last_match(2)).nil?
+
+        type = Regexp.last_match(1)
+        title = normalized_title(Regexp.last_match(2), type)
+        resource(type: type, title: title).nil?
       end
     end
 
+    # Private method: Given a title string, normalize it according to the rules
+    # used by puppet 4.10.x for file resource title normalization:
+    # https://github.com/puppetlabs/puppet/blob/4.10.x/lib/puppet/type/file.rb#L42
+    def normalized_title(title_string, type)
+      return title_string if type != 'File'
+
+      matches = title_string.match(%r{^(?<normalized_path>/|.+:/|.*[^/])/*\Z}m)
+      matches[:normalized_path] || title_string
+    end
+
     # Private method: Build the resource hash to be used used for O(1) lookups by type and title.
     # This method is called the first time the resource hash is accessed.
     def build_resource_hash
       @resource_hash = {}
       resources.each do |resource|
         @resource_hash[resource['type']] ||= {}
-        @resource_hash[resource['type']][resource['title']] = resource
 
-        if resource.key?('parameters') && resource['parameters'].key?('alias')
-          @resource_hash[resource['type']][resource['parameters']['alias']] = resource
+        title = normalized_title(resource['title'], resource['type'])
+        @resource_hash[resource['type']][title] = resource
+
+        if resource.key?('parameters')
+          @resource_hash[resource['type']][resource['parameters']['alias']] = resource if resource['parameters'].key?('alias')
+          @resource_hash[resource['type']][resource['parameters']['name']] = resource if resource['parameters'].key?('name')
         end
       end
     end

data/lib/octocatalog-diff/catalog/computed.rb

@@ -147,7 +147,8 @@ module OctocatalogDiff
             puppet_binary: @puppet_binary,
             fact_file: @builddir.fact_file,
             dir: @builddir.tempdir,
-            enc: @builddir.enc
+            enc: @builddir.enc,
+            puppet_version: puppet_version
           )
           OctocatalogDiff::CatalogUtil::Command.new(command_opts)
         end

data/lib/octocatalog-diff/catalog/puppetmaster.rb

@@ -62,16 +62,19 @@ module OctocatalogDiff
         fetch_catalog(logger)
       end
 
-      # Returns a hash of parameters for each supported version of the Puppet Server Catalog API.
+      # Returns a hash of parameters for the requested version of the Puppet Server Catalog API.
       # @return [Hash] Hash of parameters
       #
       # Note: The double escaping of the facts here is implemented to correspond to a long standing
       # bug in the Puppet code. See https://github.com/puppetlabs/puppet/pull/1818 and
       # https://docs.puppet.com/puppet/latest/http_api/http_catalog.html#parameters for explanation.
-      def puppet_catalog_api
-        {
+      def puppet_catalog_api(version)
+        api_style = {
           2 => {
             url: "https://#{@options[:puppet_master]}/#{@options[:branch]}/catalog/#{@node}",
+            headers: {
+              'Accept' => 'text/pson'
+            },
             parameters: {
               'facts_format' => 'pson',
               'facts' => CGI.escape(@facts.fudge_timestamp.without('trusted').to_pson),
@@ -80,24 +83,59 @@ module OctocatalogDiff
           },
           3 => {
             url: "https://#{@options[:puppet_master]}/puppet/v3/catalog/#{@node}",
+            headers: {
+              'Accept' => 'text/pson'
+            },
             parameters: {
               'environment' => @options[:branch],
               'facts_format' => 'pson',
               'facts' => CGI.escape(@facts.fudge_timestamp.without('trusted').to_pson),
               'transaction_uuid' => SecureRandom.uuid
             }
+          },
+          4 => {
+            url: "https://#{@options[:puppet_master]}/puppet/v4/catalog",
+            headers: {
+              'Content-Type' => 'application/json'
+            },
+            parameters: {
+              'certname' => @node,
+              'persistence' => {
+                'facts' => @options[:puppet_master_update_facts] || false,
+                'catalog' => @options[:puppet_master_update_catalog] || false
+              },
+              'environment' => @options[:branch],
+              'facts' => { 'values' => @facts.facts['values'] },
+              'options' => {
+                'prefer_requested_environment' => true,
+                'capture_logs' => false,
+                'log_level' => 'warning'
+              },
+              'transaction_uuid' => SecureRandom.uuid
+            }
           }
         }
+
+        params = api_style[version]
+        return nil if params.nil?
+
+        unless @options[:puppet_master_token].nil?
+          params[:headers]['X-Authentication'] = @options[:puppet_master_token]
+        end
+
+        params[:parameters] = params[:parameters].to_json if version >= 4
+
+        params
       end
 
       # Fetch catalog by contacting the Puppet master, sending the facts, and asking for the catalog. When the
       # catalog is returned in PSON format, parse it to JSON and then set appropriate variables.
       def fetch_catalog(logger)
         api_version = @options[:puppet_master_api_version] || DEFAULT_PUPPET_SERVER_API
-        api = puppet_catalog_api[api_version]
+        api = puppet_catalog_api(api_version)
         raise ArgumentError, "Unsupported or invalid API version #{api_version}" unless api.is_a?(Hash)
 
-        more_options = { headers: { 'Accept' => 'text/pson' }, timeout: @timeout }
+        more_options = { headers: api[:headers], timeout: @timeout }
         post_hash = api[:parameters]
 
         response = nil