octocatalog-diff 1.5.1 → 2.0.0

data/doc/requirements.md

@@ -2,10 +2,14 @@
 
 To run `octocatalog-diff` you will need these basics:
 
-- Ruby 2.0 through 2.4 (we test octocatalog-diff with Ruby 2.0, 2.1, 2.2, 2.3, and 2.4)
+- An appropriate Puppet version and [corresponding ruby version](https://puppet.com/docs/puppet/5.4/system_requirements.html)
+  - Puppet 5.x officially supports Ruby 2.4
+  - Puppet 4.x officially supports Ruby 2.1, but seems to work fine with later versions as well
+  - Puppet 3.8.7 -- we attempt to maintain compatibility in `octocatalog-diff` to facilitate upgrades even though this version is no longer supported by Puppet
+  - We don't officially support Puppet 3.8.6 or before
 - Mac OS, Linux, or other Unix-line operating system (Windows is not supported)
 - Ability to install gems, e.g. with [rbenv](https://github.com/rbenv/rbenv) or [rvm](https://rvm.io/), or root privileges to install into the system Ruby
-- Puppet agent for [Linux](https://docs.puppet.com/puppet/latest/reference/install_linux.html) or [Mac OS X](https://docs.puppet.com/puppet/latest/reference/install_osx.html), or installed as a gem (we support Puppet 3.8.7 and all versions of Puppet 4.x)
+- Puppet agent for [Linux](https://docs.puppet.com/puppet/latest/reference/install_linux.html) or [Mac OS X](https://docs.puppet.com/puppet/latest/reference/install_osx.html), or installed as a gem
 
 We recommend that you also have the following to get the most out of `octocatalog-diff`, but these are not absolute requirements:
 

data/lib/octocatalog-diff/catalog-diff/differ.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'diffy'
+require 'digest'
 require 'hashdiff'
 require 'json'
 require 'set'
@@ -11,6 +12,8 @@ require_relative '../errors'
 require_relative '../util/util'
 require_relative 'filter'
 
+HashDiff = Hashdiff unless defined? HashDiff
+
 module OctocatalogDiff
   module CatalogDiff
     # Calculate the difference between two Puppet catalogs.
@@ -263,7 +266,7 @@ module OctocatalogDiff
 
             # Handle parameters
             if k == 'parameters'
-              cleansed_param = cleanse_parameters_hash(v)
+              cleansed_param = cleanse_parameters_hash(v, resource.fetch('sensitive_parameters', []))
               hsh[k] = cleansed_param unless cleansed_param.nil? || cleansed_param.empty?
             elsif k == 'tags'
               # The order of tags is unimportant. Sort this array to avoid false diffs if order changes.
@@ -294,10 +297,13 @@ module OctocatalogDiff
         # Use diffy to get only the lines that have changed in a text object.
         # As we iterate through the diff, jump out if we have our answer: either
         # true if '=~>' finds ANY match, or false if '=&>' fails to find a match.
-        Diffy::Diff.new(old_val, new_val, context: 0).each do |line|
+        diffy_result = Diffy::Diff.new(old_val, new_val, context: 0)
+        newline_alerts = diffy_result.count { |line| line.strip == '\' }
+        diffy_result.each do |line|
           if regex.match(line.strip)
             return true if operator == '=~>'
           elsif operator == '=&>'
+            next if line.strip == '\' && newline_alerts == 2
             return false
           end
         end
@@ -334,7 +340,8 @@ module OctocatalogDiff
         #   =->  Attribute must have been removed and equal this
         #   =~>  Change must match regexp (one line of change matching is sufficient)
         #   =&>  Change must match regexp (all lines of change MUST match regexp)
-        if rule_attr =~ /\A(.+?)(=[\-\+~&]?>)(.+)/m
+        #   =s>  Change must be array and contain identical elements, ignoring order
+        if rule_attr =~ /\A(.+?)(=[\-\+~&s]?>)(.+)/m
           rule_attr = Regexp.last_match(1)
           operator = Regexp.last_match(2)
           value = Regexp.last_match(3)
@@ -355,6 +362,9 @@ module OctocatalogDiff
               raise RegexpError, "Invalid ignore regexp for #{key}: #{exc.message}"
             end
             matcher = ->(x, y) { regexp_operator_match?(operator, my_regex, x, y) }
+          elsif operator == '=s>'
+            raise ArgumentError, "Invalid ignore option for =s>, must be '='" unless value == '='
+            matcher = ->(x, y) { x.is_a?(Array) && y.is_a?(Array) && Set.new(x) == Set.new(y) }
           end
         end
 
@@ -394,6 +404,13 @@ module OctocatalogDiff
           return false unless rule[:title].casecmp(hsh[:title]).zero?
         end
 
+        # If rule[:attr] is a regular expression, handle that case here.
+        if rule[:attr].is_a?(Regexp)
+          return false unless hsh[:attr].is_a?(String)
+          return false unless rule[:attr].match(hsh[:attr])
+          return ignore_match_true(hsh, rule)
+        end
+
         # Special 'attributes': Ignore specific diff types (+ add, - remove, ~ and ! change)
         if rule[:attr] =~ /\A[\-\+~!]+\Z/
           return ignore_match_true(hsh, rule) if rule[:attr].include?(diff_type)
@@ -446,10 +463,18 @@ module OctocatalogDiff
 
       # Cleanse parameters of filtered attributes.
       # @param parameters_hash [Hash] Hash of parameters
+      # @param sensitive_parameters [Array] Array of sensitive parameters
       # @return [Hash] Cleaned parameters hash (original input hash is not altered)
-      def cleanse_parameters_hash(parameters_hash)
+      def cleanse_parameters_hash(parameters_hash, sensitive_parameters)
         result = parameters_hash.dup
 
+        # hides sensitive params. We still need to know if there's a going to
+        # be a diff, so we hash the value.
+        sensitive_parameters.each do |p|
+          md5 = Digest::MD5.hexdigest Marshal.dump(result[p])
+          result[p] = 'Sensitive [md5sum ' + md5 + ']'
+        end
+
         # 'before' and 'require' handle internal Puppet ordering but do not affect what
         # happens on the target machine. Don't consider these for the purpose of catalog diff.
         result.delete('before')

data/lib/octocatalog-diff/catalog-diff/filter.rb

@@ -2,6 +2,7 @@ require_relative '../api/v1/diff'
 require_relative 'filter/absent_file'
 require_relative 'filter/compilation_dir'
 require_relative 'filter/json'
+require_relative 'filter/single_item_array'
 require_relative 'filter/yaml'
 
 require 'stringio'
@@ -13,7 +14,7 @@ module OctocatalogDiff
       attr_accessor :logger
 
       # List the available filters here (by class name) for use in the validator method.
-      AVAILABLE_FILTERS = %w(AbsentFile CompilationDir JSON YAML).freeze
+      AVAILABLE_FILTERS = %w(AbsentFile CompilationDir JSON SingleItemArray YAML).freeze
 
       # Public: Determine whether a particular filter exists. This can be used to validate
       # a user-submitted filter.

data/lib/octocatalog-diff/catalog-diff/filter/compilation_dir.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../filter'
+require_relative '../../util/util'
 
 module OctocatalogDiff
   module CatalogDiff
@@ -35,43 +36,46 @@ module OctocatalogDiff
 
           # Check for a change where the difference in a parameter exactly corresponds to the difference in the
           # compilation directory.
-          if diff.change? && (diff.old_value.is_a?(String) || diff.new_value.is_a?(String))
-            from_before = nil
-            from_after = nil
-            from_match = false
-            to_before = nil
-            to_after = nil
-            to_match = false
+          if diff.change?
+            o = remove_compilation_dir(diff.old_value, dir2)
+            n = remove_compilation_dir(diff.new_value, dir1)
 
-            if diff.old_value =~ /^(.*)#{dir2}(.*)$/m
-              from_before = Regexp.last_match(1) || ''
-              from_after = Regexp.last_match(2) || ''
-              from_match = true
-            end
-
-            if diff.new_value =~ /^(.*)#{dir1}(.*)$/m
-              to_before = Regexp.last_match(1) || ''
-              to_after = Regexp.last_match(2) || ''
-              to_match = true
-            end
-
-            if from_match && to_match && to_before == from_before && to_after == from_after
+            if o != diff.old_value || n != diff.new_value
               message = "Resource key #{diff.type}[#{diff.title}] #{diff.structure.join(' => ')}"
-              message += ' appears to depend on catalog compilation directory. Suppressed from results.'
+              message += ' may depend on catalog compilation directory, but there may be differences.'
+              message += ' This is included in results for now, but please verify.'
               @logger.warn message
-              return true
             end
 
-            if from_match || to_match
+            if o == n
               message = "Resource key #{diff.type}[#{diff.title}] #{diff.structure.join(' => ')}"
-              message += ' may depend on catalog compilation directory, but there may be differences.'
-              message += ' This is included in results for now, but please verify.'
+              message += ' appears to depend on catalog compilation directory. Suppressed from results.'
               @logger.warn message
+              return true
             end
           end
 
           false
         end
+
+        def remove_compilation_dir(v, dir)
+          value = OctocatalogDiff::Util::Util.deep_dup(v)
+          traverse(value) do |e|
+            e.gsub!(dir, '') if e.respond_to?(:gsub!)
+          end
+          value
+        end
+
+        def traverse(a)
+          case a
+          when Array
+            a.map { |v| traverse(v, &Proc.new) }
+          when Hash
+            traverse(a.values, &Proc.new)
+          else
+            yield a
+          end
+        end
       end
     end
   end

data/lib/octocatalog-diff/catalog-diff/filter/single_item_array.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative '../filter'
+
+module OctocatalogDiff
+  module CatalogDiff
+    class Filter
+      # Filter out changes in parameters when one catalog has a parameter that's an object and
+      # the other catalog has that same parameter as an array containing the same object.
+      # For example, under this filter, the following is not a change:
+      #   catalog1: notify => "Service[foo]"
+      #   catalog2: notify => ["Service[foo]"]
+      class SingleItemArray < OctocatalogDiff::CatalogDiff::Filter
+        # Public: Implement the filter for single-item arrays whose item exactly matches the
+        # item that's not in an array in the other catalog.
+        #
+        # @param diff [OctocatalogDiff::API::V1::Diff] Difference
+        # @param _options [Hash] Additional options (there are none for this filter)
+        # @return [Boolean] true if this should be filtered out, false otherwise
+        def filtered?(diff, _options = {})
+          # Skip additions or removals - focus only on changes
+          return false unless diff.change?
+          old_value = diff.old_value
+          new_value = diff.new_value
+
+          # Skip unless there is a single-item array under consideration
+          return false unless
+            (old_value.is_a?(Array) && old_value.size == 1) ||
+            (new_value.is_a?(Array) && new_value.size == 1)
+
+          # Skip if both the old value and new value are arrays
+          return false if old_value.is_a?(Array) && new_value.is_a?(Array)
+
+          # Do comparison
+          if old_value.is_a?(Array)
+            old_value.first == new_value
+          else
+            new_value.first == old_value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/octocatalog-diff/catalog-util/builddir.rb

@@ -149,13 +149,13 @@ module OctocatalogDiff
           raise ArgumentError, 'Called install_fact_file without node, or with an empty node'
         end
 
-        facts = if options[:fact_file]
+        facts = if options[:facts].is_a?(OctocatalogDiff::Facts)
+          options[:facts].dup
+        elsif options[:fact_file]
           raise Errno::ENOENT, "Fact file #{options[:fact_file]} does not exist" unless File.file?(options[:fact_file])
           fact_file_opts = { fact_file_string: File.read(options[:fact_file]) }
           fact_file_opts[:backend] = Regexp.last_match(1).to_sym if options[:fact_file] =~ /.*\.(\w+)$/
           OctocatalogDiff::Facts.new(fact_file_opts)
-        elsif options[:facts].is_a?(OctocatalogDiff::Facts)
-          options[:facts].dup
         else
           raise ArgumentError, 'No facts passed to "install_fact_file" method'
         end

data/lib/octocatalog-diff/catalog-util/command.rb

@@ -54,9 +54,21 @@ module OctocatalogDiff
         raise ArgumentError, 'Puppet binary was not supplied' if @puppet_binary.nil?
         raise Errno::ENOENT, "Puppet binary #{@puppet_binary} doesn't exist" unless File.file?(@puppet_binary)
 
+        puppet_version = Gem::Version.new(@options[:puppet_version])
+
         # Node to compile
         cmdline = []
-        cmdline.concat ['master', '--compile', Shellwords.escape(@node)]
+        # The 'puppet master --compile' command was removed in Puppet 6.x and replaced in
+        # Puppet 6.5 with an identically functioning 'puppet catalog compile' command.
+        # From versions 6.0.0 until 6.5.0 there is no compatible invocation method.
+        if puppet_version < Gem::Version.new('6.0.0')
+          cmdline.concat ['master', '--compile', Shellwords.escape(@node)]
+        elsif puppet_version < Gem::Version.new('6.5.0')
+          raise OctocatalogDiff::Errors::PuppetVersionError,
+                'Octocatalog-diff does not support Puppet versions >= 6.0.0 and < 6.5.0'
+        else
+          cmdline.concat ['catalog', 'compile', Shellwords.escape(@node)]
+        end
 
         # storeconfigs?
         if @options[:storeconfigs]
@@ -93,11 +105,21 @@ module OctocatalogDiff
         # Some typical options for puppet
         cmdline.concat %w(
           --no-daemonize
-          --no-ca
           --color=false
-          --config_version="/bin/echo catalogscript"
         )
 
+        if puppet_version < Gem::Version.new('6.0.0')
+          # This config_version parameter causes an error when run with Puppet 6.x. Per
+          # the Puppet configuration settings docs, the below config_version argument
+          # may not actually be valid, but for backward compatibility's sake we'll keep it
+          # for the versions it has always worked with:
+          cmdline.concat ['--config_version="/bin/echo catalogscript"']
+
+          # The 'ca' configuration option was removed in Puppet 6, but we'll keep it
+          # for older versions:
+          cmdline.concat ['--no-ca']
+        end
+
         # Add environment - only make this variable if preserve_environments is used.
         # If preserve_environments is not used, the hard-coded 'production' here matches
         # up with the symlink created under the temporary directory structure.

data/lib/octocatalog-diff/catalog-util/fileresources.rb

@@ -59,7 +59,7 @@ module OctocatalogDiff
           result = []
           Regexp.last_match(1).split(/:/).map(&:strip).each do |path|
             next if path.start_with?('$')
-            result << File.expand_path(path, dir)
+            result.concat(Dir.glob(File.expand_path(path, dir)))
           end
           result
         else

data/lib/octocatalog-diff/catalog.rb

@@ -191,6 +191,8 @@ module OctocatalogDiff
       build
       raise OctocatalogDiff::Errors::CatalogError, 'Catalog does not appear to have been built' if !valid? && error_message.nil?
       raise OctocatalogDiff::Errors::CatalogError, error_message unless valid?
+      # Handle the structure returned by the /puppet/v4/catalog Puppetserver endpoint:
+      return @catalog['catalog']['resources'] if @catalog['catalog'].is_a?(Hash) && @catalog['catalog']['resources'].is_a?(Array)
       return @catalog['data']['resources'] if @catalog['data'].is_a?(Hash) && @catalog['data']['resources'].is_a?(Array)
       return @catalog['resources'] if @catalog['resources'].is_a?(Array)
       # This is a bug condition
@@ -304,20 +306,36 @@ module OctocatalogDiff
         unless res =~ /\A([\w:]+)\[(.+)\]\z/
           raise ArgumentError, "Resource #{res} is not in the expected format"
         end
-        resource(type: Regexp.last_match(1), title: Regexp.last_match(2)).nil?
+
+        type = Regexp.last_match(1)
+        title = normalized_title(Regexp.last_match(2), type)
+        resource(type: type, title: title).nil?
       end
     end
 
+    # Private method: Given a title string, normalize it according to the rules
+    # used by puppet 4.10.x for file resource title normalization:
+    # https://github.com/puppetlabs/puppet/blob/4.10.x/lib/puppet/type/file.rb#L42
+    def normalized_title(title_string, type)
+      return title_string if type != 'File'
+
+      matches = title_string.match(%r{^(?<normalized_path>/|.+:/|.*[^/])/*\Z}m)
+      matches[:normalized_path] || title_string
+    end
+
     # Private method: Build the resource hash to be used used for O(1) lookups by type and title.
     # This method is called the first time the resource hash is accessed.
     def build_resource_hash
       @resource_hash = {}
       resources.each do |resource|
         @resource_hash[resource['type']] ||= {}
-        @resource_hash[resource['type']][resource['title']] = resource
 
-        if resource.key?('parameters') && resource['parameters'].key?('alias')
-          @resource_hash[resource['type']][resource['parameters']['alias']] = resource
+        title = normalized_title(resource['title'], resource['type'])
+        @resource_hash[resource['type']][title] = resource
+
+        if resource.key?('parameters')
+          @resource_hash[resource['type']][resource['parameters']['alias']] = resource if resource['parameters'].key?('alias')
+          @resource_hash[resource['type']][resource['parameters']['name']] = resource if resource['parameters'].key?('name')
         end
       end
     end

data/lib/octocatalog-diff/catalog/computed.rb

@@ -147,7 +147,8 @@ module OctocatalogDiff
             puppet_binary: @puppet_binary,
             fact_file: @builddir.fact_file,
             dir: @builddir.tempdir,
-            enc: @builddir.enc
+            enc: @builddir.enc,
+            puppet_version: puppet_version
           )
           OctocatalogDiff::CatalogUtil::Command.new(command_opts)
         end

data/lib/octocatalog-diff/catalog/puppetmaster.rb

@@ -62,16 +62,19 @@ module OctocatalogDiff
         fetch_catalog(logger)
       end
 
-      # Returns a hash of parameters for each supported version of the Puppet Server Catalog API.
+      # Returns a hash of parameters for the requested version of the Puppet Server Catalog API.
       # @return [Hash] Hash of parameters
       #
       # Note: The double escaping of the facts here is implemented to correspond to a long standing
       # bug in the Puppet code. See https://github.com/puppetlabs/puppet/pull/1818 and
       # https://docs.puppet.com/puppet/latest/http_api/http_catalog.html#parameters for explanation.
-      def puppet_catalog_api
-        {
+      def puppet_catalog_api(version)
+        api_style = {
           2 => {
             url: "https://#{@options[:puppet_master]}/#{@options[:branch]}/catalog/#{@node}",
+            headers: {
+              'Accept' => 'text/pson'
+            },
             parameters: {
               'facts_format' => 'pson',
               'facts' => CGI.escape(@facts.fudge_timestamp.without('trusted').to_pson),
@@ -80,24 +83,59 @@ module OctocatalogDiff
           },
           3 => {
             url: "https://#{@options[:puppet_master]}/puppet/v3/catalog/#{@node}",
+            headers: {
+              'Accept' => 'text/pson'
+            },
             parameters: {
               'environment' => @options[:branch],
               'facts_format' => 'pson',
               'facts' => CGI.escape(@facts.fudge_timestamp.without('trusted').to_pson),
               'transaction_uuid' => SecureRandom.uuid
             }
+          },
+          4 => {
+            url: "https://#{@options[:puppet_master]}/puppet/v4/catalog",
+            headers: {
+              'Content-Type' => 'application/json'
+            },
+            parameters: {
+              'certname' => @node,
+              'persistence' => {
+                'facts' => @options[:puppet_master_update_facts] || false,
+                'catalog' => @options[:puppet_master_update_catalog] || false
+              },
+              'environment' => @options[:branch],
+              'facts' => { 'values' => @facts.facts['values'] },
+              'options' => {
+                'prefer_requested_environment' => true,
+                'capture_logs' => false,
+                'log_level' => 'warning'
+              },
+              'transaction_uuid' => SecureRandom.uuid
+            }
           }
         }
+
+        params = api_style[version]
+        return nil if params.nil?
+
+        unless @options[:puppet_master_token].nil?
+          params[:headers]['X-Authentication'] = @options[:puppet_master_token]
+        end
+
+        params[:parameters] = params[:parameters].to_json if version >= 4
+
+        params
       end
 
       # Fetch catalog by contacting the Puppet master, sending the facts, and asking for the catalog. When the
       # catalog is returned in PSON format, parse it to JSON and then set appropriate variables.
       def fetch_catalog(logger)
         api_version = @options[:puppet_master_api_version] || DEFAULT_PUPPET_SERVER_API
-        api = puppet_catalog_api[api_version]
+        api = puppet_catalog_api(api_version)
         raise ArgumentError, "Unsupported or invalid API version #{api_version}" unless api.is_a?(Hash)
 
-        more_options = { headers: { 'Accept' => 'text/pson' }, timeout: @timeout }
+        more_options = { headers: api[:headers], timeout: @timeout }
         post_hash = api[:parameters]
 
         response = nil