octocatalog-diff 1.5.1 → 2.0.0

data/lib/octocatalog-diff/cli.rb

@@ -11,6 +11,7 @@ require_relative 'util/util'
 require_relative 'version'
 
 require 'logger'
+require 'parallel'
 require 'socket'
 
 module OctocatalogDiff
@@ -116,16 +117,46 @@ module OctocatalogDiff
       end
 
       # Compile catalogs and do catalog-diff
-      catalog_diff = OctocatalogDiff::API::V1.catalog_diff(options.merge(logger: logger))
+      node_set = options.delete(:node)
+      node_set = [node_set] unless node_set.is_a?(Array)
+
+      # run multiple node diffs in parallel
+      catalog_diffs = if node_set.size == 1
+        [run_octocatalog_diff(node_set.first, options, logger)]
+      else
+        ::Parallel.map(node_set, in_threads: 4) { |node| run_octocatalog_diff(node, options, logger) }
+      end
+
+      # Return the resulting diff object if requested (generally for testing)
+      # or otherwise return exit code
+      return catalog_diffs.first if opts[:INTEGRATION]
+
+      all_diffs = catalog_diffs.map(&:diffs)
+
+      all_diffs.each do |diff|
+        next unless diff.any?
+        return EXITCODE_SUCCESS_WITH_DIFFS
+      end
+
+      EXITCODE_SUCCESS_NO_DIFFS
+    end
+
+    # Run the octocatalog-diff process for a given node. Return the diffs for a contribution to
+    # the final exit status.
+    # node    - String with the node
+    # options - All of the currently defined options
+    # logger  - Logger object
+    def self.run_octocatalog_diff(node, options, logger)
+      options_copy = options.merge(node: node)
+      catalog_diff = OctocatalogDiff::API::V1.catalog_diff(options_copy.merge(logger: logger))
       diffs = catalog_diff.diffs
 
       # Display diffs
-      printer_obj = OctocatalogDiff::Cli::Printer.new(options, logger)
+      printer_obj = OctocatalogDiff::Cli::Printer.new(options_copy, logger)
       printer_obj.printer(diffs, catalog_diff.from.compilation_dir, catalog_diff.to.compilation_dir)
 
-      # Return the resulting diff object if requested (generally for testing) or otherwise return exit code
-      return catalog_diff if opts[:INTEGRATION]
-      diffs.any? ? EXITCODE_SUCCESS_WITH_DIFFS : EXITCODE_SUCCESS_NO_DIFFS
+      # Return catalog-diff object.
+      catalog_diff
     end
 
     # Parse command line options with 'optparse'. Returns a hash with the parsed arguments.

data/lib/octocatalog-diff/cli/options.rb

@@ -11,7 +11,7 @@ module OctocatalogDiff
     # This class contains the option parser. 'parse_options' is the external entry point.
     class Options
       # The usage banner.
-      BANNER = 'Usage: catalog-diff -n <hostname> [-f <from environment>] [-t <to environment>]'.freeze
+      BANNER = 'Usage: catalog-diff -n <hostname>[,<hostname>...] [-f <from environment>] [-t <to environment>]'.freeze
 
       # An error class specifically for passing information to the document build task.
       class DocBuildError < RuntimeError; end
@@ -23,7 +23,6 @@ module OctocatalogDiff
 
       # Define the Option class and newoption() method for use by cli/options/*.rb files
       class Option
-        DEFAULT_WEIGHT = 999
         def self.has_weight(w) # rubocop:disable Style/PredicateName
           @weight = w
         end
@@ -38,7 +37,9 @@ module OctocatalogDiff
           elsif @weight
             @weight
           else
-            DEFAULT_WEIGHT
+            # :nocov:
+            raise ArgumentError, "Option #{name} does not have a weight specified. Add 'has_weight NNN' to control ordering."
+            # :nocov:
           end
         end
 
@@ -102,6 +103,7 @@ module OctocatalogDiff
         datatype = opts.fetch(:datatype, '')
         return option_globally_or_per_branch_string(opts) if datatype.is_a?(String)
         return option_globally_or_per_branch_array(opts) if datatype.is_a?(Array)
+        return option_globally_or_per_branch_boolean(opts) if datatype.is_a?(TrueClass) || datatype.is_a?(FalseClass)
         raise ArgumentError, "option_globally_or_per_branch not equipped to handle #{datatype.class}"
       end
 
@@ -176,6 +178,40 @@ module OctocatalogDiff
         end
       end
 
+      # See description of `option_globally_or_per_branch`. This implements the logic for a boolean value.
+      # @param :parser [OptionParser object] The OptionParser argument
+      # @param :options [Hash] Options hash being constructed; this is modified in this method.
+      # @param :cli_name [String] Name of option on command line (e.g. puppet-binary)
+      # @param :option_name [Symbol] Name of option in the options hash (e.g. :puppet_binary)
+      # @param :desc [String] Description of option on the command line; will have "for the XX branch" appended
+      def self.option_globally_or_per_branch_boolean(opts)
+        parser = opts.fetch(:parser)
+        options = opts.fetch(:options)
+        cli_name = opts.fetch(:cli_name)
+        option_name = opts.fetch(:option_name)
+        desc = opts.fetch(:desc)
+
+        flag = cli_name
+        from_option = "from_#{option_name}".to_sym
+        to_option = "to_#{option_name}".to_sym
+        parser.on("--[no-]#{flag}", "#{desc} globally") do |x|
+          translated = translate_option(opts[:translator], x)
+          options[to_option] = translated
+          options[from_option] = translated
+          post_process(opts[:post_process], options)
+        end
+        parser.on("--[no-]to-#{flag}", "#{desc} for the to branch") do |x|
+          translated = translate_option(opts[:translator], x)
+          options[to_option] = translated
+          post_process(opts[:post_process], options)
+        end
+        parser.on("--[no-]from-#{flag}", "#{desc} for the from branch") do |x|
+          translated = translate_option(opts[:translator], x)
+          options[from_option] = translated
+          post_process(opts[:post_process], options)
+        end
+      end
+
       # If a validator was provided, run the validator on the supplied value. The validator is expected to
       # throw an error if there is a problem. Note that the validator runs *before* the translator if both
       # a validator and translator are supplied.

data/lib/octocatalog-diff/cli/options/hostname.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 # Set hostname, which is used to look up facts in PuppetDB, and in the header of diff display.
+# This option can recieve a single hostname, or a comma separated list of
+# multiple hostnames, which are split into an Array. Multiple hostnames do not
+# work with the `catalog-only` or `bootstrap-then-exit` options.
 # @param parser [OptionParser object] The OptionParser argument
 # @param options [Hash] Options hash being constructed; this is modified in this method.
 
@@ -8,8 +11,16 @@ OctocatalogDiff::Cli::Options::Option.newoption(:hostname) do
   has_weight 1
 
   def parse(parser, options)
-    parser.on('--hostname HOSTNAME', '-n', 'Use PuppetDB facts from last run of hostname') do |hostname|
-      options[:node] = hostname
+    parser.on(
+      '--hostname HOSTNAME1[,HOSTNAME2[,...]]',
+      '-n',
+      'Use PuppetDB facts from last run of a hostname or a comma separated list of multiple hostnames'
+    ) do |hostname|
+      options[:node] = if hostname.include?(',')
+        hostname.split(',')
+      else
+        hostname
+      end
     end
   end
 end

data/lib/octocatalog-diff/cli/options/pe_enc_token_file.rb

@@ -12,7 +12,7 @@ OctocatalogDiff::Cli::Options::Option.newoption(:pe_enc_token_file) do
   def parse(parser, options)
     parser.on('--pe-enc-token-file PATH', 'Path containing token for PE node classifier, relative or absolute') do |x|
       proposed_token_path = x.start_with?('/') ? x : File.join(options[:basedir], x)
-      raise Errno::ENOENT, "Provided token (#{proposed_token_path}) does not exist" unless File.file?(proposed_token_path)
+      raise Errno::ENOENT, "Provided PE ENC token (#{proposed_token_path}) does not exist" unless File.file?(proposed_token_path)
       options[:pe_enc_token] = File.read(proposed_token_path)
     end
   end

data/lib/octocatalog-diff/cli/options/puppet_master_api_version.rb

@@ -14,8 +14,8 @@ OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_api_version) do
       options: options,
       cli_name: 'puppet-master-api-version',
       option_name: 'puppet_master_api_version',
-      desc: 'Puppet Master API version (2 for Puppet 3.x, 3 for Puppet 4.x)',
-      validator: ->(x) { x =~ /^[23]$/ || raise(ArgumentError, 'Only API versions 2 and 3 are supported') },
+      desc: 'Puppet Master API version (2 for Puppet 3.x, 3 for Puppet 4.x, 4 for Puppet Server >= 6.3.0)',
+      validator: ->(x) { x =~ /^[234]$/ || raise(ArgumentError, 'Only API versions 2, 3, and 4 are supported') },
       translator: ->(x) { x.to_i }
     )
   end

data/lib/octocatalog-diff/cli/options/puppet_master_token.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Specify a PE RBAC token used to authenticate to Puppetserver for v4
+# catalog API calls.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_token) do
+  has_weight 310
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: '',
+      cli_name: 'puppet-master-token',
+      option_name: 'puppet_master_token',
+      desc: 'PE RBAC token to authenticate to the Puppetserver API v4'
+    )
+  end
+end

data/lib/octocatalog-diff/cli/options/puppet_master_token_file.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Specify a path to a file containing a PE RBAC token used to authenticate to the
+# Puppetserver for a v4 catalog API call.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_token_file) do
+  has_weight 300
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: '',
+      cli_name: 'puppet-master-token-file',
+      option_name: 'puppet_master_token_file',
+      desc: 'File containing PE RBAC token to authenticate to the Puppetserver API v4',
+      translator: ->(x) { x.start_with?('/', '~') ? x : File.join(options[:basedir], x) },
+      post_process: lambda do |opts|
+        %w(to from).each do |prefix|
+          fileopt = "#{prefix}_puppet_master_token_file".to_sym
+          tokenopt = "#{prefix}_puppet_master_token".to_sym
+
+          tokenfile = opts[fileopt]
+          next if tokenfile.nil?
+
+          raise(Errno::ENOENT, "Token file #{tokenfile} is not readable") unless File.readable?(tokenfile)
+
+          token = File.read(tokenfile).strip
+          opts[tokenopt] ||= token
+        end
+      end
+    )
+  end
+end

data/lib/octocatalog-diff/cli/options/puppet_master_update_catalog.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Specify if, when using the Puppetserver v4 catalog API, the Puppetserver should
+# update the catalog in PuppetDB.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_update_catalog) do
+  has_weight 320
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: false,
+      cli_name: 'puppet-master-update-catalog',
+      option_name: 'puppet_master_update_catalog',
+      desc: 'Update catalog in PuppetDB when using Puppetmaster API version 4'
+    )
+  end
+end

data/lib/octocatalog-diff/cli/options/puppet_master_update_facts.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Specify if, when using the Puppetserver v4 catalog API, the Puppetserver should
+# update the facts in PuppetDB.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_update_facts) do
+  has_weight 320
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: false,
+      cli_name: 'puppet-master-update-facts',
+      option_name: 'puppet_master_update_facts',
+      desc: 'Update facts in PuppetDB when using Puppetmaster API version 4'
+    )
+  end
+end

data/lib/octocatalog-diff/cli/options/puppetdb_package_inventory.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# When pulling facts from PuppetDB in a Puppet Enterprise environment, also include
+# the Puppet Enterprise Package Inventory data in the fact results, if available.
+# Generally you should not need to specify this, but including the package inventory
+# data will produce a more accurate set of input facts for environments using
+# package inventory.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppetdb_package_inventory) do
+  has_weight 150
+
+  def parse(parser, options)
+    parser.on('--[no-]puppetdb-package-inventory', 'Include Puppet Enterprise package inventory data, if found') do |x|
+      options[:puppetdb_package_inventory] = x
+    end
+  end
+end

data/lib/octocatalog-diff/cli/options/puppetdb_token.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Specify the PE RBAC token to access the PuppetDB API. Refer to
+# https://puppet.com/docs/pe/latest/rbac/rbac_token_auth_intro.html#generate-a-token-using-puppet-access
+# for details on generating and obtaining a token. Use this option to specify the text
+# of the token. (Use --puppetdb-token-file to read the content of the token from a file.)
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppetdb_token) do
+  has_weight 310
+
+  def parse(parser, options)
+    parser.on('--puppetdb-token TOKEN', 'Token to access the PuppetDB API') do |token|
+      options[:puppetdb_token] = token
+    end
+  end
+end

data/lib/octocatalog-diff/cli/options/puppetdb_token_file.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Specify the PE RBAC token to access the PuppetDB API. Refer to
+# https://puppet.com/docs/pe/latest/rbac/rbac_token_auth_intro.html#generate-a-token-using-puppet-access
+# for details on generating and obtaining a token. Use this option to specify the text
+# in a file, to read the content of the token from the file.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppetdb_token_file) do
+  has_weight 310
+
+  def parse(parser, options)
+    parser.on('--puppetdb-token-file PATH', 'Path containing token for PuppetDB API, relative or absolute') do |x|
+      proposed_token_path = x.start_with?('/') ? x : File.join(options[:basedir], x)
+      unless File.file?(proposed_token_path)
+        raise Errno::ENOENT, "Provided PuppetDB API token (#{proposed_token_path}) does not exist"
+      end
+      options[:puppetdb_token] = File.read(proposed_token_path)
+    end
+  end
+end

data/lib/octocatalog-diff/facts/puppetdb.rb

@@ -36,6 +36,7 @@ module OctocatalogDiff
         exception_class = nil
         exception_message = nil
         obj_to_return = nil
+        packages = nil
         (retries + 1).times do
           begin
             result = puppetdb.get(uri)
@@ -61,8 +62,48 @@ module OctocatalogDiff
             exception_message = "Fact retrieval failed for node #{node} from PuppetDB (#{exc.message})"
           end
         end
-        return obj_to_return unless obj_to_return.nil?
-        raise exception_class, exception_message
+
+        raise exception_class, exception_message if obj_to_return.nil?
+
+        return obj_to_return if puppetdb_api_version < 4 || (!options[:puppetdb_package_inventory])
+
+        (retries + 1).times do
+          begin
+            result = puppetdb.get("/pdb/query/v4/package-inventory/#{node}")
+            packages = {}
+            result.each do |pkg|
+              key = "#{pkg['package_name']}+#{pkg['provider']}"
+              # Need to handle the situation where a package has multiple versions installed.
+              # The _puppet_inventory_1 hash lists them separated by "; ".
+              if packages.key?(key)
+                packages[key]['version'] += "; #{pkg['version']}"
+              else
+                packages[key] = pkg
+              end
+            end
+            break
+          rescue OctocatalogDiff::Errors::PuppetDBConnectionError => exc
+            exception_class = OctocatalogDiff::Errors::FactSourceError
+            exception_message = "Package inventory retrieval failed (#{exc.class}) (#{exc.message})"
+          # This is not expected to occur, but we'll leave it just in case. A query to package-inventory
+          # for a non-existant node returns a 200 OK with an empty list of packages:
+          rescue OctocatalogDiff::Errors::PuppetDBNodeNotFoundError
+            packages = {}
+          rescue OctocatalogDiff::Errors::PuppetDBGenericError => exc
+            exception_class = OctocatalogDiff::Errors::FactRetrievalError
+            exception_message = "Package inventory retrieval failed for node #{node} from PuppetDB (#{exc.message})"
+          end
+        end
+
+        raise exception_class, exception_message if packages.nil?
+
+        unless packages.empty?
+          obj_to_return['values']['_puppet_inventory_1'] = {
+            'packages' => packages.values.map { |pkg| [pkg['package_name'], pkg['version'], pkg['provider']] }
+          }
+        end
+
+        obj_to_return
       end
     end
   end

data/lib/octocatalog-diff/puppetdb.rb

@@ -42,6 +42,7 @@ module OctocatalogDiff
     # @param :puppetdb_ssl_client_p12 [String] pkcs12-encoded client key and certificate
     # @param :puppetdb_ssl_client_password [String] Path to file containing password for SSL client key (any format)
     # @param :puppetdb_ssl_client_auth [Boolean] Override the client-auth that is guessed from parameters
+    # @param :puppetdb_token [String] PE RBAC token to authenticate to PuppetDB API
     # @param :timeout [Integer] Connection timeout for PuppetDB (default=10)
     def initialize(options = {})
       @connections =
@@ -107,7 +108,10 @@ module OctocatalogDiff
         ].join('')
 
         begin
-          more_options = { headers: { 'Accept' => 'application/json' }, timeout: @timeout }
+          headers = { 'Accept' => 'application/json' }
+          headers['X-Authentication'] = @options[:puppetdb_token] if @options[:puppetdb_token]
+          more_options = { headers: headers, timeout: @timeout }
+
           if connection[:username] || connection[:password]
             more_options[:basic_auth] = { username: connection[:username], password: connection[:password] }
           end

data/lib/octocatalog-diff/util/parallel.rb

@@ -129,22 +129,26 @@ module OctocatalogDiff
 
         # Waiting for children and handling results
         while pidmap.any?
-          this_pid, exit_obj = Process.wait2(0)
-          next unless this_pid && pidmap.key?(this_pid)
-          index = pidmap[this_pid][:index]
-          exitstatus = exit_obj.exitstatus
-          raise "PID=#{this_pid} exited abnormally: #{exit_obj.inspect}" if exitstatus.nil?
-          raise "PID=#{this_pid} exited with status #{exitstatus}" unless exitstatus.zero?
-
-          input = File.read(File.join(ipc_tempdir, "#{this_pid}.dat"))
-          result[index] = Marshal.load(input) # rubocop:disable Security/MarshalLoad
-          time_delta = Time.now - pidmap[this_pid][:start_time]
-          pidmap.delete(this_pid)
-
-          logger.debug "PID=#{this_pid} completed in #{time_delta} seconds, #{input.length} bytes"
-
-          next if result[index].status
-          return result[index].exception
+          pidmap.each do |pid|
+            status = Process.waitpid2(pid[0], Process::WNOHANG)
+            next if status.nil?
+            this_pid, exit_obj = status
+            next unless this_pid && pidmap.key?(this_pid)
+            index = pidmap[this_pid][:index]
+            exitstatus = exit_obj.exitstatus
+            raise "PID=#{this_pid} exited abnormally: #{exit_obj.inspect}" if exitstatus.nil?
+            raise "PID=#{this_pid} exited with status #{exitstatus}" unless exitstatus.zero?
+
+            input = File.read(File.join(ipc_tempdir, "#{this_pid}.dat"))
+            result[index] = Marshal.load(input) # rubocop:disable Security/MarshalLoad
+            time_delta = Time.now - pidmap[this_pid][:start_time]
+            pidmap.delete(this_pid)
+
+            logger.debug "PID=#{this_pid} completed in #{time_delta} seconds, #{input.length} bytes"
+
+            next if result[index].status
+            return result[index].exception
+          end
         end
 
         logger.debug 'All child processes completed with no exceptions raised'