octocatalog-diff 0.6.1 → 1.0.0

data/lib/octocatalog-diff/{catalog-diff/cli → cli}/options/storeconfigs.rb

@@ -3,7 +3,7 @@
 # Set storeconfigs (integration with PuppetDB for collected resources)
 # @param parser [OptionParser object] The OptionParser argument
 # @param options [Hash] Options hash being constructed; this is modified in this method.
-OctocatalogDiff::CatalogDiff::Cli::Options::Option.newoption(:storeconfigs) do
+OctocatalogDiff::Cli::Options::Option.newoption(:storeconfigs) do
   has_weight 220
 
   def parse(parser, options)

data/lib/octocatalog-diff/{catalog-diff/cli → cli}/options/suppress_absent_file_details.rb

@@ -3,9 +3,10 @@
 # If enabled, this option will suppress changes to certain attributes of a file, if the
 # file is specified to be 'absent' in the target catalog. Suppressed changes in this case
 # include user, group, mode, and content, because a removed file has none of those.
+# <i>This option is DEPRECATED; please use <code>--filters AbsentFile</code> instead.</i>
 # @param parser [OptionParser object] The OptionParser argument
 # @param options [Hash] Options hash being constructed; this is modified in this method.
-OctocatalogDiff::CatalogDiff::Cli::Options::Option.newoption(:suppress_absent_file_details) do
+OctocatalogDiff::Cli::Options::Option.newoption(:suppress_absent_file_details) do
   has_weight 600
 
   def parse(parser, options)

data/lib/octocatalog-diff/{catalog-diff/cli → cli}/options/to_from_branch.rb

@@ -4,7 +4,7 @@
 # the current contents of the base code directory without any git checkouts.
 # @param parser [OptionParser object] The OptionParser argument
 # @param options [Hash] Options hash being constructed; this is modified in this method.
-OctocatalogDiff::CatalogDiff::Cli::Options::Option.newoption(:to_from_branch) do
+OctocatalogDiff::Cli::Options::Option.newoption(:to_from_branch) do
   has_weight 20
 
   def parse(parser, options)

data/lib/octocatalog-diff/{catalog-diff/cli → cli}/options/validate_references.rb

@@ -5,7 +5,7 @@
 # parameters are to be checked.
 # @param parser [OptionParser object] The OptionParser argument
 # @param options [Hash] Options hash being constructed; this is modified in this method.
-OctocatalogDiff::CatalogDiff::Cli::Options::Option.newoption(:validate_references) do
+OctocatalogDiff::Cli::Options::Option.newoption(:validate_references) do
   has_weight 205
 
   def parse(parser, options)

data/lib/octocatalog-diff/cli/printer.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative '../catalog-diff/display'
+require_relative '../errors'
+
+module OctocatalogDiff
+  class Cli
+    # Wrapper around OctocatalogDiff::CatalogDiff::Display to set the options and
+    # output to a file or the screen depending on selection.
+    class Printer
+      # Constructor
+      # @param options [Hash] Options from cli/options
+      # @param logger [Logger] Logger object
+      def initialize(options, logger)
+        @options = options
+        @logger = logger
+      end
+
+      # The method to call externally, passing in diffs. This takes the appropriate action
+      # based on options, which is either to write the result into an output file, or print
+      # the result on STDOUT. Does not return anything.
+      # @param diffs [Array<Diffs>] Array of differences
+      # @param from_dir [String] Directory in which "from" catalog was compiled
+      # @param to_dir [String] Directory in which "to" catalog was compiled
+      def printer(diffs, from_dir = nil, to_dir = nil)
+        unless diffs.is_a?(Array)
+          raise ArgumentError, "printer() expects an array, not #{diffs.class}"
+        end
+        display_opts = @options.merge(compilation_from_dir: from_dir, compilation_to_dir: to_dir)
+        diff_text = OctocatalogDiff::CatalogDiff::Display.output(diffs, display_opts, @logger)
+        if @options[:output_file].nil?
+          puts diff_text unless diff_text.empty?
+        else
+          output_to_file(diff_text)
+        end
+      end
+
+      private
+
+      # Output to a file, handling errors related to writing files.
+      # @param diff_in [String|Array] Text to write to file
+      def output_to_file(diff_in)
+        diff_text = diff_in.is_a?(Array) ? diff_in.join("\n") : diff_in
+        File.open(@options[:output_file], 'w') { |f| f.write(diff_text) }
+        @logger.info "Wrote diff to #{@options[:output_file]}"
+      rescue Errno::ENOENT, Errno::EACCES, Errno::EISDIR => exc
+        @logger.error "Cannot write to #{@options[:output_file]}: #{exc}"
+        raise OctocatalogDiff::Errors::PrinterError, "Cannot write to #{@options[:output_file]}: #{exc}"
+      end
+    end
+  end
+end

data/lib/octocatalog-diff/errors.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module OctocatalogDiff
+  # Contains error classes raised by this gem
+  class Errors
+    # Error classes for handled configuration file errors
+    class ConfigurationFileNotFoundError < RuntimeError; end
+    class ConfigurationFileContentError < RuntimeError; end
+
+    # Error classes for building catalogs
+    class BootstrapError < RuntimeError; end
+    class CatalogError < RuntimeError; end
+    class PuppetVersionError < RuntimeError; end
+    class ReferenceValidationError < RuntimeError; end
+    class GitCheckoutError < RuntimeError; end
+
+    # Error classes for retrieving facts
+    class FactSourceError < RuntimeError; end
+    class FactRetrievalError < RuntimeError; end
+
+    # Errors for PuppetDB
+    class PuppetDBNodeNotFoundError < RuntimeError; end
+    class PuppetDBGenericError < RuntimeError; end
+    class PuppetDBConnectionError < RuntimeError; end
+
+    # Errors for Puppet Enterprise
+    class PEClassificationError < RuntimeError; end
+
+    # Miscellanous catalog-diff errors
+    class DifferError < RuntimeError; end
+    class PrinterError < RuntimeError; end
+  end
+end

data/lib/octocatalog-diff/facts.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'errors'
 require_relative 'facts/json'
 require_relative 'facts/yaml'
 require_relative 'facts/puppetdb'
@@ -119,9 +120,5 @@ module OctocatalogDiff
         @facts['values'][key] = value
       end
     end
-
-    # Separate classes to handle errors we throw explicitly
-    class FactSourceError < RuntimeError; end
-    class FactRetrievalError < RuntimeError; end
   end
 end

data/lib/octocatalog-diff/facts/puppetdb.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../errors'
 require_relative '../facts'
 require_relative '../puppetdb'
 require 'yaml'
@@ -42,21 +43,21 @@ module OctocatalogDiff
             result.map { |x| facts[x['name']] = x['value'] }
             if facts.empty?
               message = "Unable to retrieve facts for node #{node} from PuppetDB (empty or nil)!"
-              raise OctocatalogDiff::Facts::FactRetrievalError, message
+              raise OctocatalogDiff::Errors::FactRetrievalError, message
             end
 
             # Create a structure compatible with YAML fact files.
             obj_to_return = { 'name' => node, 'values' => {} }
             facts.each { |k, v| obj_to_return['values'][k.sub(/^::/, '')] = v }
             break # Not return, to avoid LocalJumpError in Ruby 2.2
-          rescue OctocatalogDiff::PuppetDB::ConnectionError => exc
-            exception_class = OctocatalogDiff::Facts::FactSourceError
+          rescue OctocatalogDiff::Errors::PuppetDBConnectionError => exc
+            exception_class = OctocatalogDiff::Errors::FactSourceError
             exception_message = "Fact retrieval failed (#{exc.class}) (#{exc.message})"
-          rescue OctocatalogDiff::PuppetDB::NotFoundError => exc
-            exception_class = OctocatalogDiff::Facts::FactRetrievalError
+          rescue OctocatalogDiff::Errors::PuppetDBNodeNotFoundError => exc
+            exception_class = OctocatalogDiff::Errors::FactRetrievalError
             exception_message = "Node #{node} not found in PuppetDB (#{exc.message})"
-          rescue OctocatalogDiff::PuppetDB::PuppetDBError => exc
-            exception_class = OctocatalogDiff::Facts::FactRetrievalError
+          rescue OctocatalogDiff::Errors::PuppetDBGenericError => exc
+            exception_class = OctocatalogDiff::Errors::FactRetrievalError
             exception_message = "Fact retrieval failed for node #{node} from PuppetDB (#{exc.message})"
           end
         end

data/lib/octocatalog-diff/puppetdb.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'errors'
 require_relative 'util/httparty'
 
 require 'uri'
@@ -14,11 +15,6 @@ URI::HTTPS.const_set(:DEFAULT_PORT, 8081)
 module OctocatalogDiff
   # A standard way to connect to PuppetDB from the various scripts in this repository.
   class PuppetDB
-    # This class raises errors for certain handled problems
-    class NotFoundError < RuntimeError; end
-    class PuppetDBError < RuntimeError; end
-    class ConnectionError < RuntimeError; end
-
     # Allow connections to be read (used in tests for now)
     attr_reader :connections
 
@@ -87,7 +83,7 @@ module OctocatalogDiff
     def get(path)
       _get(path)
     rescue Net::OpenTimeout, Errno::ECONNREFUSED => exc
-      raise ConnectionError, "#{exc.class} connecting to PuppetDB (need VPN on?): #{exc.message}"
+      raise OctocatalogDiff::Errors::PuppetDBConnectionError, "#{exc.class} connecting to PuppetDB (need VPN on?): #{exc.message}"
     end
 
     private
@@ -120,8 +116,8 @@ module OctocatalogDiff
 
           # Handle all non-200's from PuppetDB
           unless response[:code] == 200
-            raise NotFoundError, "404 - #{response[:error]}" if response[:code] == 404
-            raise PuppetDBError, "#{response[:code]} - #{response[:error]}"
+            raise OctocatalogDiff::Errors::PuppetDBNodeNotFoundError, "404 - #{response[:error]}" if response[:code] == 404
+            raise OctocatalogDiff::Errors::PuppetDBGenericError, "#{response[:code]} - #{response[:error]}"
           end
 
           # PuppetDB can return 'Not Found' as a string with a 200 response code
@@ -129,7 +125,7 @@ module OctocatalogDiff
 
           # PuppetDB can also return an error message in a 200; we'll call this a 500
           if response.key?(:error)
-            raise PuppetDBError, "500 - #{response[:error]}"
+            raise OctocatalogDiff::Errors::PuppetDBGenericError, "500 - #{response[:error]}"
           end
 
           # If we get here without raising an error, it will fall out of the begin/rescue

data/lib/octocatalog-diff/util/catalogs.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+require 'yaml'
+require_relative '../catalog'
+require_relative '../errors'
+require_relative 'parallel'
+
+module OctocatalogDiff
+  module Util
+    # Helper class to construct catalogs, performing all necessary steps such as
+    # bootstrapping directories, installing facts, and running puppet.
+    class Catalogs
+      # Constructor
+      # @param options [Hash] Options
+      # @param logger [Logger] Logger object
+      def initialize(options, logger)
+        @options = options
+        @logger = logger
+        @catalogs = nil
+        raise '@logger must not be nil' if @logger.nil?
+      end
+
+      # Compile catalogs. This handles building both the old and new catalog (in parallel) and returns
+      # only when both catalogs have been built.
+      # @return [Hash] { :from => [OctocatalogDiff::Catalog], :to => [OctocatalogDiff::Catalog] }
+      def catalogs
+        @catalogs ||= build_catalog_parallelizer
+      end
+
+      # Handles the "bootstrap then exit" option, which bootstraps directories but
+      # exits without compiling catalogs.
+      def bootstrap_then_exit
+        @logger.debug('Begin bootstrap_then_exit')
+        OctocatalogDiff::CatalogUtil::Bootstrap.bootstrap_directory_parallelizer(@options, @logger)
+        @logger.debug('Success bootstrap_then_exit')
+        @logger.info('Successfully completed --bootstrap-then-exit action')
+      end
+
+      private
+
+      # Parallelizes bootstrapping of directories and building catalogs.
+      # @return [Hash] { :from => OctocatalogDiff::Catalog, :to => OctocatalogDiff::Catalog }
+      def build_catalog_parallelizer
+        # Construct parallel tasks. The array supplied to OctocatalogDiff::Util::Parallel is the task portion
+        # of each of the tuples in catalog_tasks.
+        catalog_tasks = build_catalog_tasks
+
+        # Update any tasks for catalogs that do not need to be compiled. This is the case when --catalog-only
+        # is specified and only one catalog is to be built. This will change matching catalog tasks to the 'noop' type.
+        catalog_tasks.map! do |x|
+          if @options["#{x[0]}_catalog".to_sym] == '-'
+            x[1].args[:backend] = :noop
+          elsif @options["#{x[0]}_catalog".to_sym].is_a?(String)
+            x[1].args[:json] = File.read(@options["#{x[0]}_catalog".to_sym])
+            x[1].args[:backend] = :json
+          end
+          x
+        end
+
+        # Initialize the objects for each parallel task. Initializing the object is very fast and does not actually
+        # build the catalog.
+        result = {}
+        catalog_tasks.each do |x|
+          result[x[0]] = OctocatalogDiff::Catalog.new(x[1].args)
+          @logger.debug "Initialized #{result[x[0]].builder} for #{x[0]}-catalog"
+        end
+
+        # Disable --compare-file-text if either (or both) of the chosen backends do not support it
+        if @options.fetch(:compare_file_text, false)
+          result.each do |_key, val|
+            next unless val.convert_file_resources == false
+            @logger.debug "Disabling --compare-file-text; not supported by #{val.builder}"
+            @options[:compare_file_text] = false
+            catalog_tasks.map! do |x|
+              x[1].args[:compare_file_text] = false
+              x
+            end
+            break
+          end
+        end
+
+        # Inject the starting object into the catalog tasks
+        catalog_tasks.map! do |x|
+          x[1].args[:object] = result[x[0]]
+          x
+        end
+
+        # Execute the parallelized catalog builds
+        passed_catalog_tasks = catalog_tasks.map { |x| x[1] }
+        parallel_catalogs = OctocatalogDiff::Util::Parallel.run_tasks(passed_catalog_tasks, @logger, @options[:parallel])
+
+        # If the catalogs array is empty at this point, there is an unexpected size mismatch. This should
+        # never happen, but test for it anyway.
+        unless parallel_catalogs.size == catalog_tasks.size
+          # :nocov:
+          raise "BUG: mismatch catalog_result (#{parallel_catalogs.size} vs #{catalog_tasks.size})"
+          # :nocov:
+        end
+
+        # Construct result hash. Will eventually be in the format
+        # { :from => OctocatalogDiff::Catalog, :to => OctocatalogDiff::Catalog }
+
+        # Analyze the results from parallel run.
+        catalog_tasks.each do |x|
+          # The `parallel_catalog_obj` is a OctocatalogDiff::Util::Parallel::Result. Get the first element from
+          # the parallel_catalogs output.
+          parallel_catalog_obj = parallel_catalogs.shift
+
+          # Add the result to the 'result' hash
+          add_parallel_result(result, parallel_catalog_obj, x)
+        end
+
+        # Things have succeeded if the :to and :from catalogs exist at this point. If not, things have
+        # failed, and an exception should be thrown.
+        return result if result.key?(:to) && result.key?(:from)
+
+        # This is believed to be a bug condition.
+        # :nocov:
+        raise OctocatalogDiff::Errors::CatalogError, 'One or more catalogs failed to compile.'
+        # :nocov:
+      end
+
+      # Get catalog compilation tasks.
+      # @return [Array<[key, task]>] Catalog tasks
+      def build_catalog_tasks
+        [:from, :to].map do |key|
+          # These are arguments to OctocatalogDiff::Util::Parallel::Task. In most cases the arguments
+          # of OctocatalogDiff::Util::Parallel::Task are taken directly from options, but there are
+          # some defaults or otherwise-named options that must be set here.
+          args = @options.merge(
+            tag: key.to_s,
+            branch: @options["#{key}_env".to_sym] || '-',
+            bootstrapped_dir: @options["bootstrapped_#{key}_dir".to_sym],
+            basedir: @options[:basedir],
+            compare_file_text: @options.fetch(:compare_file_text, true),
+            retry_failed_catalog: @options.fetch(:retry_failed_catalog, 0),
+            parser: @options["parser_#{key}".to_sym]
+          )
+
+          # If any options are in the form of 'to_SOMETHING' or 'from_SOMETHING', this sets the option to
+          # 'SOMETHING' for the catalog if it matches this key. For example, when compiling the 'to' catalog
+          # when an option of :to_some_arg => 'foo', this sets :some_arg => foo, and deletes :to_some_arg and
+          # :from_some_arg.
+          @options.keys.select { |x| x.to_s =~ /^(to|from)_/ }.each do |opt_key|
+            args[opt_key.to_s.sub(/^(to|from)_/, '').to_sym] = @options[opt_key] if opt_key.to_s.start_with?(key.to_s)
+            args.delete(opt_key)
+          end
+
+          # The task is a OctocatalogDiff::Util::Parallel::Task object that contains the method to execute,
+          # validator method, text description, and arguments to provide when calling the method.
+          task = OctocatalogDiff::Util::Parallel::Task.new(
+            method: method(:build_catalog),
+            validator: method(:catalog_validator),
+            validator_args: { task: key },
+            description: "build_catalog for #{@options["#{key}_env".to_sym]}",
+            args: args
+          )
+
+          # The format of `catalog_tasks` will be a tuple, where the first element is the key
+          # (e.g. :to or :from) and the second element is the OctocatalogDiff::Util::Parallel::Task object.
+          [key, task]
+        end.compact
+      end
+
+      # Given a result from the 'parallel' run and a corresponding (key,task) tuple, add valid
+      # catalogs to the 'result' hash and throw errors for invalid catalogs.
+      # @param result [Hash] Result hash for build_catalog_parallelizer (may be modified)
+      # @param parallel_catalog_obj [OctocatalogDiff::Util::Parallel::Result] Parallel catalog result
+      # @param key_task_tuple [Array<key, task>] Key, task tuple
+      def add_parallel_result(result, parallel_catalog_obj, key_task_tuple)
+        # Expand the tuple into variables
+        key, task = key_task_tuple
+
+        # For reporting purposes, get the branch name.
+        branch = task.args[:branch]
+
+        # Check the result of the parallel run on this object.
+        if parallel_catalog_obj.status.nil?
+          # The compile was killed because another task failed.
+          @logger.warn "Catalog compile for #{branch} was aborted due to another failure"
+
+        elsif parallel_catalog_obj.output.is_a?(OctocatalogDiff::Catalog)
+          # The result is a catalog, but we do not know if it was successfully compiled
+          # until we test the validity.
+          catalog = parallel_catalog_obj.output
+          if catalog.valid?
+            # The catalog was successfully compiled.
+            result[key] = parallel_catalog_obj.output
+          else
+            # The catalog failed, but a catalog object was returned so that better error reporting
+            # can take place. In this error reporting, we will replace 'Error:' with '[Puppet Error]'
+            # and remove the compilation directory (which is a tmpdir) to reveal only the relative
+            # path to the files involved.
+            dir = catalog.compilation_dir || ''
+            dir_regex = Regexp.new(Regexp.escape(dir) + '/environments/[^/]+/')
+            error_display = catalog.error_message.split("\n").map do |line|
+              line.sub(/^Error:/, '[Puppet Error]').gsub(dir_regex, '')
+            end.join("\n")
+            message = "Catalog for #{branch} failed to compile due to errors:\n#{error_display}"
+            raise OctocatalogDiff::Errors::CatalogError, message
+          end
+        else
+          # Something unhandled went wrong, and an exception was thrown. Reveal a generic message.
+          msg = parallel_catalog_obj.exception.message
+          message = "Catalog for '#{key}' (#{branch}) failed to compile with #{parallel_catalog_obj.exception.class}: #{msg}"
+          message += "\n" + parallel_catalog_obj.exception.backtrace.map { |x| "   #{x}" }.join("\n") if @options[:debug]
+          raise OctocatalogDiff::Errors::CatalogError, message
+        end
+      end
+
+      # Performs the steps necessary to build a catalog.
+      # @param opts [Hash] Options hash
+      # @return [Hash] { :rc => exit code, :catalog => Catalog as JSON string }
+      def build_catalog(opts, logger = @logger)
+        logger.debug("Setting up Puppet catalog build for #{opts[:branch]}")
+        catalog = opts[:object]
+        logger.debug("Catalog for #{opts[:branch]} will be built with #{catalog.builder}")
+        time_start = Time.now
+        catalog.build(logger)
+        time_it_took = Time.now - time_start
+        retries_str = " retries = #{catalog.retries}" if catalog.retries.is_a?(Fixnum)
+        time_str = "in #{time_it_took} seconds#{retries_str}"
+        status_str = catalog.valid? ? 'successfully built' : 'failed'
+        logger.debug "Catalog for #{opts[:branch]} #{status_str} with #{catalog.builder} #{time_str}"
+        catalog
+      end
+
+      # Validate a catalog in the parallel execution
+      # @param catalog [OctocatalogDiff::Catalog] Catalog object
+      # @param logger [Logger] Logger object (presently unused)
+      # @param args [Hash] Additional arguments set specifically for validator
+      # @return [Boolean] true if catalog is valid, false otherwise
+      def catalog_validator(catalog = nil, _logger = @logger, args = {})
+        return false unless catalog.is_a?(OctocatalogDiff::Catalog)
+        catalog.validate_references if args[:task] == :to
+        catalog.valid?
+      end
+    end
+  end
+end