octocatalog-diff 1.5.2 → 2.1.0

data/lib/octocatalog-diff/cli.rb

@@ -11,6 +11,7 @@ require_relative 'util/util'
 require_relative 'version'
 
 require 'logger'
+require 'parallel'
 require 'socket'
 
 module OctocatalogDiff
@@ -43,7 +44,8 @@ module OctocatalogDiff
       display_datatype_changes: true,
       parallel: true,
       suppress_absent_file_details: true,
-      hiera_path: 'hieradata'
+      hiera_path: 'hieradata',
+      use_lcs: true
     }.freeze
 
     # This method is the one to call externally. It is possible to specify alternate
@@ -116,16 +118,46 @@ module OctocatalogDiff
       end
 
       # Compile catalogs and do catalog-diff
-      catalog_diff = OctocatalogDiff::API::V1.catalog_diff(options.merge(logger: logger))
+      node_set = options.delete(:node)
+      node_set = [node_set] unless node_set.is_a?(Array)
+
+      # run multiple node diffs in parallel
+      catalog_diffs = if node_set.size == 1
+        [run_octocatalog_diff(node_set.first, options, logger)]
+      else
+        ::Parallel.map(node_set, in_threads: 4) { |node| run_octocatalog_diff(node, options, logger) }
+      end
+
+      # Return the resulting diff object if requested (generally for testing)
+      # or otherwise return exit code
+      return catalog_diffs.first if opts[:INTEGRATION]
+
+      all_diffs = catalog_diffs.map(&:diffs)
+
+      all_diffs.each do |diff|
+        next unless diff.any?
+        return EXITCODE_SUCCESS_WITH_DIFFS
+      end
+
+      EXITCODE_SUCCESS_NO_DIFFS
+    end
+
+    # Run the octocatalog-diff process for a given node. Return the diffs for a contribution to
+    # the final exit status.
+    # node    - String with the node
+    # options - All of the currently defined options
+    # logger  - Logger object
+    def self.run_octocatalog_diff(node, options, logger)
+      options_copy = options.merge(node: node)
+      catalog_diff = OctocatalogDiff::API::V1.catalog_diff(options_copy.merge(logger: logger))
       diffs = catalog_diff.diffs
 
       # Display diffs
-      printer_obj = OctocatalogDiff::Cli::Printer.new(options, logger)
+      printer_obj = OctocatalogDiff::Cli::Printer.new(options_copy, logger)
       printer_obj.printer(diffs, catalog_diff.from.compilation_dir, catalog_diff.to.compilation_dir)
 
-      # Return the resulting diff object if requested (generally for testing) or otherwise return exit code
-      return catalog_diff if opts[:INTEGRATION]
-      diffs.any? ? EXITCODE_SUCCESS_WITH_DIFFS : EXITCODE_SUCCESS_NO_DIFFS
+      # Return catalog-diff object.
+      catalog_diff
     end
 
     # Parse command line options with 'optparse'. Returns a hash with the parsed arguments.

data/lib/octocatalog-diff/cli/options.rb

@@ -11,7 +11,7 @@ module OctocatalogDiff
     # This class contains the option parser. 'parse_options' is the external entry point.
     class Options
       # The usage banner.
-      BANNER = 'Usage: catalog-diff -n <hostname> [-f <from environment>] [-t <to environment>]'.freeze
+      BANNER = 'Usage: catalog-diff -n <hostname>[,<hostname>...] [-f <from environment>] [-t <to environment>]'.freeze
 
       # An error class specifically for passing information to the document build task.
       class DocBuildError < RuntimeError; end
@@ -103,6 +103,7 @@ module OctocatalogDiff
         datatype = opts.fetch(:datatype, '')
         return option_globally_or_per_branch_string(opts) if datatype.is_a?(String)
         return option_globally_or_per_branch_array(opts) if datatype.is_a?(Array)
+        return option_globally_or_per_branch_boolean(opts) if datatype.is_a?(TrueClass) || datatype.is_a?(FalseClass)
         raise ArgumentError, "option_globally_or_per_branch not equipped to handle #{datatype.class}"
       end
 
@@ -177,6 +178,40 @@ module OctocatalogDiff
         end
       end
 
+      # See description of `option_globally_or_per_branch`. This implements the logic for a boolean value.
+      # @param :parser [OptionParser object] The OptionParser argument
+      # @param :options [Hash] Options hash being constructed; this is modified in this method.
+      # @param :cli_name [String] Name of option on command line (e.g. puppet-binary)
+      # @param :option_name [Symbol] Name of option in the options hash (e.g. :puppet_binary)
+      # @param :desc [String] Description of option on the command line; will have "for the XX branch" appended
+      def self.option_globally_or_per_branch_boolean(opts)
+        parser = opts.fetch(:parser)
+        options = opts.fetch(:options)
+        cli_name = opts.fetch(:cli_name)
+        option_name = opts.fetch(:option_name)
+        desc = opts.fetch(:desc)
+
+        flag = cli_name
+        from_option = "from_#{option_name}".to_sym
+        to_option = "to_#{option_name}".to_sym
+        parser.on("--[no-]#{flag}", "#{desc} globally") do |x|
+          translated = translate_option(opts[:translator], x)
+          options[to_option] = translated
+          options[from_option] = translated
+          post_process(opts[:post_process], options)
+        end
+        parser.on("--[no-]to-#{flag}", "#{desc} for the to branch") do |x|
+          translated = translate_option(opts[:translator], x)
+          options[to_option] = translated
+          post_process(opts[:post_process], options)
+        end
+        parser.on("--[no-]from-#{flag}", "#{desc} for the from branch") do |x|
+          translated = translate_option(opts[:translator], x)
+          options[from_option] = translated
+          post_process(opts[:post_process], options)
+        end
+      end
+
       # If a validator was provided, run the validator on the supplied value. The validator is expected to
       # throw an error if there is a problem. Note that the validator runs *before* the translator if both
       # a validator and translator are supplied.

data/lib/octocatalog-diff/cli/options/compare_file_text.rb

@@ -15,3 +15,21 @@ OctocatalogDiff::Cli::Options::Option.newoption(:compare_file_text) do
     end
   end
 end
+
+# Sometimes there is a particular file resource for which the file text
+# comparison is not desired, while not disabling that option globally. Similar
+# to --ignore_tags, it's possible to tag the file resource and exempt it from
+# the --compare_file_text checks.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:compare_file_text_ignore_tags) do
+  has_weight 415
+
+  def parse(parser, options)
+    description = 'Tags that exclude a file resource from text comparison'
+    parser.on('--compare-file-text-ignore-tags STRING1[,STRING2[,...]]', Array, description) do |x|
+      options[:compare_file_text_ignore_tags] ||= []
+      options[:compare_file_text_ignore_tags].concat x
+    end
+  end
+end

data/lib/octocatalog-diff/cli/options/hostname.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 # Set hostname, which is used to look up facts in PuppetDB, and in the header of diff display.
+# This option can recieve a single hostname, or a comma separated list of
+# multiple hostnames, which are split into an Array. Multiple hostnames do not
+# work with the `catalog-only` or `bootstrap-then-exit` options.
 # @param parser [OptionParser object] The OptionParser argument
 # @param options [Hash] Options hash being constructed; this is modified in this method.
 
@@ -8,8 +11,16 @@ OctocatalogDiff::Cli::Options::Option.newoption(:hostname) do
   has_weight 1
 
   def parse(parser, options)
-    parser.on('--hostname HOSTNAME', '-n', 'Use PuppetDB facts from last run of hostname') do |hostname|
-      options[:node] = hostname
+    parser.on(
+      '--hostname HOSTNAME1[,HOSTNAME2[,...]]',
+      '-n',
+      'Use PuppetDB facts from last run of a hostname or a comma separated list of multiple hostnames'
+    ) do |hostname|
+      options[:node] = if hostname.include?(',')
+        hostname.split(',')
+      else
+        hostname
+      end
     end
   end
 end

data/lib/octocatalog-diff/cli/options/puppet_master_api_version.rb

@@ -14,8 +14,8 @@ OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_api_version) do
       options: options,
       cli_name: 'puppet-master-api-version',
       option_name: 'puppet_master_api_version',
-      desc: 'Puppet Master API version (2 for Puppet 3.x, 3 for Puppet 4.x)',
-      validator: ->(x) { x =~ /^[23]$/ || raise(ArgumentError, 'Only API versions 2 and 3 are supported') },
+      desc: 'Puppet Master API version (2 for Puppet 3.x, 3 for Puppet 4.x, 4 for Puppet Server >= 6.3.0)',
+      validator: ->(x) { x =~ /^[234]$/ || raise(ArgumentError, 'Only API versions 2, 3, and 4 are supported') },
       translator: ->(x) { x.to_i }
     )
   end

data/lib/octocatalog-diff/cli/options/puppet_master_token.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Specify a PE RBAC token used to authenticate to Puppetserver for v4
+# catalog API calls.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_token) do
+  has_weight 310
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: '',
+      cli_name: 'puppet-master-token',
+      option_name: 'puppet_master_token',
+      desc: 'PE RBAC token to authenticate to the Puppetserver API v4'
+    )
+  end
+end

data/lib/octocatalog-diff/cli/options/puppet_master_token_file.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Specify a path to a file containing a PE RBAC token used to authenticate to the
+# Puppetserver for a v4 catalog API call.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_token_file) do
+  has_weight 300
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: '',
+      cli_name: 'puppet-master-token-file',
+      option_name: 'puppet_master_token_file',
+      desc: 'File containing PE RBAC token to authenticate to the Puppetserver API v4',
+      translator: ->(x) { x.start_with?('/', '~') ? x : File.join(options[:basedir], x) },
+      post_process: lambda do |opts|
+        %w(to from).each do |prefix|
+          fileopt = "#{prefix}_puppet_master_token_file".to_sym
+          tokenopt = "#{prefix}_puppet_master_token".to_sym
+
+          tokenfile = opts[fileopt]
+          next if tokenfile.nil?
+
+          raise(Errno::ENOENT, "Token file #{tokenfile} is not readable") unless File.readable?(tokenfile)
+
+          token = File.read(tokenfile).strip
+          opts[tokenopt] ||= token
+        end
+      end
+    )
+  end
+end

data/lib/octocatalog-diff/cli/options/puppet_master_update_catalog.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Specify if, when using the Puppetserver v4 catalog API, the Puppetserver should
+# update the catalog in PuppetDB.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_update_catalog) do
+  has_weight 320
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: false,
+      cli_name: 'puppet-master-update-catalog',
+      option_name: 'puppet_master_update_catalog',
+      desc: 'Update catalog in PuppetDB when using Puppetmaster API version 4'
+    )
+  end
+end

data/lib/octocatalog-diff/cli/options/puppet_master_update_facts.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Specify if, when using the Puppetserver v4 catalog API, the Puppetserver should
+# update the facts in PuppetDB.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_update_facts) do
+  has_weight 320
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: false,
+      cli_name: 'puppet-master-update-facts',
+      option_name: 'puppet_master_update_facts',
+      desc: 'Update facts in PuppetDB when using Puppetmaster API version 4'
+    )
+  end
+end

data/lib/octocatalog-diff/cli/options/puppetdb_package_inventory.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# When pulling facts from PuppetDB in a Puppet Enterprise environment, also include
+# the Puppet Enterprise Package Inventory data in the fact results, if available.
+# Generally you should not need to specify this, but including the package inventory
+# data will produce a more accurate set of input facts for environments using
+# package inventory.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppetdb_package_inventory) do
+  has_weight 150
+
+  def parse(parser, options)
+    parser.on('--[no-]puppetdb-package-inventory', 'Include Puppet Enterprise package inventory data, if found') do |x|
+      options[:puppetdb_package_inventory] = x
+    end
+  end
+end

data/lib/octocatalog-diff/cli/options/use_lcs.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Configures using the Longest common subsequence (LCS) algorithm to determine differences in arrays
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:use_lcs) do
+  has_weight 250
+
+  def parse(parser, options)
+    parser.on('--[no-]use-lcs', 'Use the LCS algorithm to determine differences in arrays') do |x|
+      options[:use_lcs] = x
+    end
+  end
+end

data/lib/octocatalog-diff/facts/json.rb

@@ -14,8 +14,19 @@ module OctocatalogDiff
       # @return [Hash] Facts
       def self.fact_retriever(options = {}, node = '')
         facts = ::JSON.parse(options.fetch(:fact_file_string))
-        node = facts.fetch('fqdn', 'unknown.node') if node.empty?
-        { 'name' => node, 'values' => facts }
+
+        if facts.keys.include?('name') && facts.keys.include?('values') && facts['values'].is_a?(Hash)
+          # If you saved the output of something like
+          # `puppet facts find $(hostname)` the structure will already be a
+          # {'name' => <fqdn>, 'values' => <hash of facts>}. We do nothing
+          # here because we don't want to double-encode.
+        else
+          facts = { 'name' => node, 'values' => facts }
+        end
+
+        facts['name'] = node unless node.empty?
+        facts['name'] = facts['values'].fetch('fqdn', 'unknown.node') if facts['name'].empty?
+        facts
       end
     end
   end

data/lib/octocatalog-diff/facts/puppetdb.rb

@@ -36,6 +36,7 @@ module OctocatalogDiff
         exception_class = nil
         exception_message = nil
         obj_to_return = nil
+        packages = nil
         (retries + 1).times do
           begin
             result = puppetdb.get(uri)
@@ -61,8 +62,48 @@ module OctocatalogDiff
             exception_message = "Fact retrieval failed for node #{node} from PuppetDB (#{exc.message})"
           end
         end
-        return obj_to_return unless obj_to_return.nil?
-        raise exception_class, exception_message
+
+        raise exception_class, exception_message if obj_to_return.nil?
+
+        return obj_to_return if puppetdb_api_version < 4 || (!options[:puppetdb_package_inventory])
+
+        (retries + 1).times do
+          begin
+            result = puppetdb.get("/pdb/query/v4/package-inventory/#{node}")
+            packages = {}
+            result.each do |pkg|
+              key = "#{pkg['package_name']}+#{pkg['provider']}"
+              # Need to handle the situation where a package has multiple versions installed.
+              # The _puppet_inventory_1 hash lists them separated by "; ".
+              if packages.key?(key)
+                packages[key]['version'] += "; #{pkg['version']}"
+              else
+                packages[key] = pkg
+              end
+            end
+            break
+          rescue OctocatalogDiff::Errors::PuppetDBConnectionError => exc
+            exception_class = OctocatalogDiff::Errors::FactSourceError
+            exception_message = "Package inventory retrieval failed (#{exc.class}) (#{exc.message})"
+          # This is not expected to occur, but we'll leave it just in case. A query to package-inventory
+          # for a non-existant node returns a 200 OK with an empty list of packages:
+          rescue OctocatalogDiff::Errors::PuppetDBNodeNotFoundError
+            packages = {}
+          rescue OctocatalogDiff::Errors::PuppetDBGenericError => exc
+            exception_class = OctocatalogDiff::Errors::FactRetrievalError
+            exception_message = "Package inventory retrieval failed for node #{node} from PuppetDB (#{exc.message})"
+          end
+        end
+
+        raise exception_class, exception_message if packages.nil?
+
+        unless packages.empty?
+          obj_to_return['values']['_puppet_inventory_1'] = {
+            'packages' => packages.values.map { |pkg| [pkg['package_name'], pkg['version'], pkg['provider']] }
+          }
+        end
+
+        obj_to_return
       end
     end
   end

data/lib/octocatalog-diff/util/parallel.rb

@@ -129,22 +129,26 @@ module OctocatalogDiff
 
         # Waiting for children and handling results
         while pidmap.any?
-          this_pid, exit_obj = Process.wait2(0)
-          next unless this_pid && pidmap.key?(this_pid)
-          index = pidmap[this_pid][:index]
-          exitstatus = exit_obj.exitstatus
-          raise "PID=#{this_pid} exited abnormally: #{exit_obj.inspect}" if exitstatus.nil?
-          raise "PID=#{this_pid} exited with status #{exitstatus}" unless exitstatus.zero?
-
-          input = File.read(File.join(ipc_tempdir, "#{this_pid}.dat"))
-          result[index] = Marshal.load(input) # rubocop:disable Security/MarshalLoad
-          time_delta = Time.now - pidmap[this_pid][:start_time]
-          pidmap.delete(this_pid)
-
-          logger.debug "PID=#{this_pid} completed in #{time_delta} seconds, #{input.length} bytes"
-
-          next if result[index].status
-          return result[index].exception
+          pidmap.each do |pid|
+            status = Process.waitpid2(pid[0], Process::WNOHANG)
+            next if status.nil?
+            this_pid, exit_obj = status
+            next unless this_pid && pidmap.key?(this_pid)
+            index = pidmap[this_pid][:index]
+            exitstatus = exit_obj.exitstatus
+            raise "PID=#{this_pid} exited abnormally: #{exit_obj.inspect}" if exitstatus.nil?
+            raise "PID=#{this_pid} exited with status #{exitstatus}" unless exitstatus.zero?
+
+            input = File.read(File.join(ipc_tempdir, "#{this_pid}.dat"))
+            result[index] = Marshal.load(input) # rubocop:disable Security/MarshalLoad
+            time_delta = Time.now - pidmap[this_pid][:start_time]
+            pidmap.delete(this_pid)
+
+            logger.debug "PID=#{this_pid} completed in #{time_delta} seconds, #{input.length} bytes"
+
+            next if result[index].status
+            return result[index].exception
+          end
         end
 
         logger.debug 'All child processes completed with no exceptions raised'