octocatalog-diff 0.6.1 → 1.0.0

data/lib/octocatalog-diff/catalog-util/enc/pe.rb

@@ -2,6 +2,7 @@
 
 require_relative 'pe/v1'
 require_relative '../../util/httparty'
+require_relative '../../errors'
 require_relative '../facts'
 
 module OctocatalogDiff
@@ -10,9 +11,6 @@ module OctocatalogDiff
       # Support the Puppet Enterprise classification API.
       # Documentation: https://docs.puppet.com/pe/latest/nc_index.html
       class PE
-        # Error class that can be caught
-        class ClassificationError < RuntimeError; end
-
         # Allow the main ENC object to retrieve these values
         attr_reader :content, :error_message
 
@@ -69,7 +67,7 @@ module OctocatalogDiff
           begin
             @content = @api.result(response[:parsed], logger)
             @error_message = nil
-          rescue OctocatalogDiff::CatalogUtil::ENC::PE::ClassificationError => exc
+          rescue OctocatalogDiff::Errors::PEClassificationError => exc
             @error_message = exc.message
             logger.error "PE ENC failed: #{exc.message}"
             return
@@ -87,7 +85,7 @@ module OctocatalogDiff
           begin
             result = facts_obj.facts
             logger.debug "Success retrieving facts for #{@node} from #{self.class}"
-          rescue OctocatalogDiff::Facts::FactRetrievalError, OctocatalogDiff::Facts::FactSourceError => exc
+          rescue OctocatalogDiff::Errors::FactRetrievalError, OctocatalogDiff::Errors::FactSourceError => exc
             @content = nil
             @error_message = "Fact retrieval failed: #{exc.class} - #{exc.message}"
             logger.error @error_message

data/lib/octocatalog-diff/catalog-util/enc/pe/v1.rb

@@ -4,6 +4,8 @@ require 'json'
 require 'uri'
 require 'yaml'
 
+require_relative '../../../errors'
+
 module OctocatalogDiff
   module CatalogUtil
     class ENC
@@ -50,7 +52,7 @@ module OctocatalogDiff
             %w(classes parameters).each do |required_key|
               next if parsed[required_key]
               logger.debug parsed.keys.inspect
-              raise OctocatalogDiff::CatalogUtil::ENC::PE::ClassificationError, "Response missing: #{required_key}"
+              raise OctocatalogDiff::Errors::PEClassificationError, "Response missing: #{required_key}"
             end
 
             obj = { 'classes' => parsed['classes'], 'parameters' => parsed['parameters'] }

data/lib/octocatalog-diff/catalog-util/git.rb

@@ -6,15 +6,13 @@ require 'rugged'
 require 'shellwords'
 require 'tempfile'
 
+require_relative '../errors'
+
 module OctocatalogDiff
   module CatalogUtil
     # Class to perform a git checkout (via 'git archive') of a branch from the base git
     # directory into another targeted directory.
     class Git
-      # Trapped errors
-      class GitCheckoutError < RuntimeError
-      end
-
       # Check out a branch via 'git archive' from one directory into another.
       # @param options [Hash] Options hash:
       #          - :branch => Branch name to check out
@@ -28,27 +26,39 @@ module OctocatalogDiff
         logger = options.fetch(:logger)
 
         # Validate parameters
-        raise GitCheckoutError, "Source directory #{dir.inspect} does not exist" if dir.nil? || !File.directory?(dir)
-        raise GitCheckoutError, "Target directory #{path.inspect} does not exist" if dir.nil? || !File.directory?(path)
-
-        # To get the options working correctly (-o pipefail in particular) this needs to run under
-        # bash. It's just creating a script, rather than figuring out all the shell escapes...
-        begin
-          tmp_script = Tempfile.new(['git-checkout', '.sh'])
-          tmp_script.write "#!/bin/bash\n"
-          tmp_script.write "set -euf -o pipefail\n"
-          tmp_script.write "git archive --format=tar #{Shellwords.escape(branch)} | \\\n"
-          tmp_script.write "  ( cd #{Shellwords.escape(path)} && tar -xf - )\n"
-          tmp_script.close
-          FileUtils.chmod 0o755, tmp_script.path
+        if dir.nil? || !File.directory?(dir)
+          raise OctocatalogDiff::Errors::GitCheckoutError, "Source directory #{dir.inspect} does not exist"
+        end
+        if path.nil? || !File.directory?(path)
+          raise OctocatalogDiff::Errors::GitCheckoutError, "Target directory #{path.inspect} does not exist"
+        end
 
-          logger.debug("Begin git archive #{dir}:#{branch} -> #{path}")
-          output, status = Open3.capture2e(tmp_script.path, chdir: dir)
-          raise GitCheckoutError, "Git archive #{branch}->#{path} failed: #{output}" unless status.exitstatus.zero?
-          logger.debug("Success git archive #{dir}:#{branch}")
-        ensure
-          FileUtils.rm_f tmp_script.path if File.exist?(tmp_script.path)
+        # Create and execute checkout script
+        script = create_git_checkout_script(branch, path)
+        logger.debug("Begin git archive #{dir}:#{branch} -> #{path}")
+        output, status = Open3.capture2e(script, chdir: dir)
+        unless status.exitstatus.zero?
+          raise OctocatalogDiff::Errors::GitCheckoutError, "Git archive #{branch}->#{path} failed: #{output}"
         end
+        logger.debug("Success git archive #{dir}:#{branch}")
+      end
+
+      # Create the temporary file used to interact with the git command line.
+      # To get the options working correctly (-o pipefail in particular) this needs to run under
+      # bash. It's just creating a script, rather than figuring out all the shell escapes...
+      # @param branch [String] Branch name
+      # @param path [String] Target directory
+      # @return [String] Name of script
+      def self.create_git_checkout_script(branch, path)
+        tmp_script = Tempfile.new(['git-checkout', '.sh'])
+        tmp_script.write "#!/bin/bash\n"
+        tmp_script.write "set -euf -o pipefail\n"
+        tmp_script.write "git archive --format=tar #{Shellwords.escape(branch)} | \\\n"
+        tmp_script.write "  ( cd #{Shellwords.escape(path)} && tar -xf - )\n"
+        tmp_script.close
+        FileUtils.chmod 0o755, tmp_script.path
+        at_exit { FileUtils.rm_f tmp_script.path if File.exist?(tmp_script.path) }
+        tmp_script.path
       end
 
       # Determine the SHA of origin/master (or any other branch really) in the git repo
@@ -58,7 +68,9 @@ module OctocatalogDiff
       def self.branch_sha(options = {})
         branch = options.fetch(:branch)
         dir = options.fetch(:basedir)
-        raise GitCheckoutError, "Git directory #{dir.inspect} does not exist" if dir.nil? || !File.directory?(dir)
+        if dir.nil? || !File.directory?(dir)
+          raise Errno::ENOENT, "Git directory #{dir.inspect} does not exist"
+        end
         repo = Rugged::Repository.new(dir)
         repo.branches[branch].target_id
       end

data/lib/octocatalog-diff/catalog.rb

@@ -9,6 +9,7 @@ require_relative 'catalog/noop'
 require_relative 'catalog/puppetdb'
 require_relative 'catalog/puppetmaster'
 require_relative 'catalog-util/fileresources'
+require_relative 'errors'
 
 module OctocatalogDiff
   # This class represents a catalog. Generation of the catalog is handled via one of the
@@ -18,11 +19,6 @@ module OctocatalogDiff
     # Readable
     attr_reader :built, :catalog, :catalog_json
 
-    # Error classes that we can throw
-    class PuppetVersionError < RuntimeError; end
-    class CatalogError < RuntimeError; end
-    class ReferenceValidationError < RuntimeError; end
-
     # Constructor
     # @param :backend [Symbol] If set, this will force a backend
     # @param :json [String] JSON catalog content (will avoid running Puppet to compile catalog)
@@ -159,8 +155,8 @@ module OctocatalogDiff
     # This is a compatibility layer for the resources, which are in a different place in Puppet 3.x and Puppet 4.x
     # @return [Array] Resource array
     def resources
-      raise CatalogError, 'Catalog does not appear to have been built' if !valid? && error_message.nil?
-      raise CatalogError, error_message unless valid?
+      raise OctocatalogDiff::Errors::CatalogError, 'Catalog does not appear to have been built' if !valid? && error_message.nil?
+      raise OctocatalogDiff::Errors::CatalogError, error_message unless valid?
       return @catalog['data']['resources'] if @catalog['data'].is_a?(Hash) && @catalog['data']['resources'].is_a?(Array)
       return @catalog['resources'] if @catalog['resources'].is_a?(Array)
       # This is a bug condition
@@ -183,7 +179,7 @@ module OctocatalogDiff
     end
 
     # Determine if all of the (before, notify, require, subscribe) targets are actually in the catalog.
-    # Raise a ReferenceValidationError for any found to be missing.
+    # Raise a OctocatalogDiff::Errors::ReferenceValidationError for any found to be missing.
     # Uses @options[:validate_references] to influence which references are checked.
     def validate_references
       # Skip out early if no reference validation has been requested.
@@ -223,7 +219,7 @@ module OctocatalogDiff
       formatted_references.flatten!
       plural = formatted_references.size == 1 ? '' : 's'
       errors = formatted_references.join('; ')
-      raise ReferenceValidationError, "Catalog has broken reference#{plural}: #{errors}"
+      raise OctocatalogDiff::Errors::ReferenceValidationError, "Catalog has broken reference#{plural}: #{errors}"
     end
 
     private

data/lib/octocatalog-diff/catalog/computed.rb

@@ -89,7 +89,15 @@ module OctocatalogDiff
       def cleanup_checkout_dir(checkout_dir, logger)
         return unless File.directory?(checkout_dir)
         logger.debug("Cleaning up temporary directory #{checkout_dir}")
-        FileUtils.remove_entry_secure checkout_dir
+        # Sometimes this seems to break when handling the recursive removal when running under
+        # a parallel environment. Trap and ignore the errors here if we don't care about them.
+        begin
+          FileUtils.remove_entry_secure checkout_dir
+          # :nocov:
+        rescue Errno::ENOTEMPTY, Errno::ENOENT => exc
+          logger.debug "cleanup_checkout_dir(#{checkout_dir}) logged #{exc.class} - this can be ignored"
+          # :nocov:
+        end
       end
 
       # Private method: Bootstrap a directory

data/lib/octocatalog-diff/catalog/puppetdb.rb

@@ -3,6 +3,7 @@
 require 'json'
 require 'stringio'
 
+require_relative '../errors'
 require_relative '../puppetdb'
 
 module OctocatalogDiff
@@ -67,11 +68,11 @@ module OctocatalogDiff
             # Set the other variables
             @catalog_json = ::JSON.generate(@catalog)
             @error_message = nil
-          rescue OctocatalogDiff::PuppetDB::ConnectionError => exc
+          rescue OctocatalogDiff::Errors::PuppetDBConnectionError => exc
             @error_message = "Catalog retrieval failed (#{exc.class}) (#{exc.message})"
-          rescue OctocatalogDiff::PuppetDB::NotFoundError => exc
+          rescue OctocatalogDiff::Errors::PuppetDBNodeNotFoundError => exc
             @error_message = "Node #{node} not found in PuppetDB (#{exc.message})"
-          rescue OctocatalogDiff::PuppetDB::PuppetDBError => exc
+          rescue OctocatalogDiff::Errors::PuppetDBGenericError => exc
             @error_message = "Catalog retrieval failed for node #{node} from PuppetDB (#{exc.message})"
           rescue ::JSON::GeneratorError => exc
             @error_message = "Failed to generate result from PuppetDB as JSON (#{exc.message})"

data/lib/octocatalog-diff/cli.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+require_relative 'api/v1'
+require_relative 'catalog-util/cached_master_directory'
+require_relative 'cli/diffs'
+require_relative 'cli/options'
+require_relative 'cli/printer'
+require_relative 'errors'
+require_relative 'util/catalogs'
+require_relative 'version'
+
+require 'logger'
+require 'socket'
+
+module OctocatalogDiff
+  # This is the CLI for catalog-diff. It's responsible for parsing the command line
+  # arguments and then handing off to appropriate methods to perform the catalog-diff.
+  class Cli
+    # Version number
+    VERSION = OctocatalogDiff::Version::VERSION
+
+    # Exit codes
+    EXITCODE_SUCCESS_NO_DIFFS = 0
+    EXITCODE_FAILURE = 1
+    EXITCODE_SUCCESS_WITH_DIFFS = 2
+
+    # The default type+title+attribute to ignore in catalog-diff.
+    DEFAULT_IGNORES = [
+      { type: 'Class' } # Don't care about classes themselves, only what they actually do!
+    ].freeze
+
+    # The default options.
+    DEFAULT_OPTIONS = {
+      from_env: 'origin/master',
+      to_env: '.',
+      colors: true,
+      debug: false,
+      quiet: false,
+      format: :color_text,
+      display_source_file_line: false,
+      compare_file_text: true,
+      display_datatype_changes: true,
+      parallel: true,
+      suppress_absent_file_details: true,
+      hiera_path: 'hieradata'
+    }.freeze
+
+    # This method is the one to call externally. It is possible to specify alternate
+    # command line arguments, for testing.
+    # @param argv [Array] Use specified arguments (defaults to ARGV)
+    # @param logger [Logger] Logger object
+    # @param opts [Hash] Additional options
+    # @return [Fixnum] Exit code: 0=no diffs, 1=something went wrong, 2=worked but there are diffs
+    def self.cli(argv = ARGV, logger = Logger.new(STDERR), opts = {})
+      # Save a copy of argv to print out later in debugging
+      argv_save = argv.dup
+
+      # Are there additional ARGV to munge, e.g. that have been supplied in the options from a
+      # configuration file?
+      if opts.key?(:additional_argv)
+        raise ArgumentError, ':additional_argv must be array!' unless opts[:additional_argv].is_a?(Array)
+        argv.concat opts[:additional_argv]
+      end
+
+      # Parse command line
+      options = parse_opts(argv)
+
+      # Additional options from hard-coded specified options. These are only processed if
+      # there are not already values defined from command line options.
+      # Note: do NOT use 'options[k] ||= v' here because if the value of options[k] is boolean(false)
+      # it will then be overridden. Whereas the intent is to define values only for those keys that don't exist.
+      opts.each { |k, v| options[k] = v unless options.key?(k) }
+      veto_options = %w(enc header hiera_config include_tags)
+      veto_options.each { |x| options.delete(x.to_sym) if options["no_#{x}".to_sym] }
+      options[:ignore].concat opts.fetch(:additional_ignores, [])
+
+      # Incorporate default options where needed.
+      # Note: do NOT use 'options[k] ||= v' here because if the value of options[k] is boolean(false)
+      # it will then be overridden. Whereas the intent is to define values only for those keys that don't exist.
+      DEFAULT_OPTIONS.each { |k, v| options[k] = v unless options.key?(k) }
+      veto_with_none_options = %w(hiera_path hiera_path_strip)
+      veto_with_none_options.each { |x| options.delete(x.to_sym) if options[x.to_sym] == :none }
+
+      # Fact and ENC overrides come in here - 'options' is modified
+      setup_fact_overrides(options)
+      setup_enc_overrides(options)
+
+      # Configure the logger and logger.debug initial information
+      # 'logger' is modified and used
+      setup_logger(logger, options, argv_save)
+
+      # --catalog-only is a special case that compiles the catalog for the "to" branch
+      # and then exits, without doing any 'diff' whatsoever. Support that option.
+      return catalog_only(logger, options) if options[:catalog_only]
+
+      # Set up the cached master directory - maintain it, adjust options if needed. However, if we
+      # are getting the 'from' catalog from PuppetDB, then don't do this.
+      unless options[:cached_master_dir].nil? || options[:from_puppetdb]
+        OctocatalogDiff::CatalogUtil::CachedMasterDirectory.run(options, logger)
+      end
+
+      # bootstrap_then_exit is a special case that only prepares directories and does not
+      # depend on facts. This happens within the 'catalogs' object, since bootstrapping and
+      # preparing catalogs are tightly coupled operations. However this does not actually
+      # build catalogs.
+      if options[:bootstrap_then_exit]
+        catalogs_obj = OctocatalogDiff::Util::Catalogs.new(options, logger)
+        return bootstrap_then_exit(logger, catalogs_obj)
+      end
+
+      # Compile catalogs and do catalog-diff
+      catalog_diff = OctocatalogDiff::API::V1.catalog_diff(options.merge(logger: logger))
+      diffs = catalog_diff.diffs
+
+      # Display diffs
+      printer_obj = OctocatalogDiff::Cli::Printer.new(options, logger)
+      printer_obj.printer(diffs, catalog_diff.from.compilation_dir, catalog_diff.to.compilation_dir)
+
+      # Return the resulting diff object if requested (generally for testing) or otherwise return exit code
+      return catalog_diff if opts[:INTEGRATION]
+      diffs.any? ? EXITCODE_SUCCESS_WITH_DIFFS : EXITCODE_SUCCESS_NO_DIFFS
+    end
+
+    # Parse command line options with 'optparse'. Returns a hash with the parsed arguments.
+    # @param argv [Array] Command line arguments (MUST be specified)
+    # @return [Hash] Options
+    def self.parse_opts(argv)
+      options = { ignore: DEFAULT_IGNORES.dup }
+      Options.parse_options(argv, options)
+    end
+
+    # Generic overrides
+    def self.setup_overrides(key, options)
+      o = options["#{key}_in".to_sym]
+      return unless o.is_a?(Array)
+      return unless o.any?
+      options[key] ||= []
+      options[key].concat o.map { |x| OctocatalogDiff::API::V1::Override.create_from_input(x) }
+    end
+
+    # Fact overrides come in here
+    def self.setup_fact_overrides(options)
+      setup_overrides(:from_fact_override, options)
+      setup_overrides(:to_fact_override, options)
+    end
+
+    # ENC parameter overrides come in here
+    def self.setup_enc_overrides(options)
+      setup_overrides(:from_enc_override, options)
+      setup_overrides(:to_enc_override, options)
+    end
+
+    # Helper method: Configure and setup logger
+    def self.setup_logger(logger, options, argv_save)
+      # Configure the logger
+      logger.level = Logger::INFO
+      logger.level = Logger::DEBUG if options[:debug]
+      logger.level = Logger::ERROR if options[:quiet]
+
+      # Some debugging information up front
+      version_display = ENV['OCTOCATALOG_DIFF_CUSTOM_VERSION'] || VERSION
+      logger.debug "Running octocatalog-diff #{version_display} with ruby #{RUBY_VERSION}"
+      logger.debug "Command line arguments: #{argv_save.inspect}"
+      logger.debug "Running on host #{Socket.gethostname} (#{RUBY_PLATFORM})"
+    end
+
+    # Compile the catalog only
+    def self.catalog_only(logger, options)
+      opts = options.merge(logger: logger)
+      to_catalog = OctocatalogDiff::API::V1.catalog(opts)
+
+      # If the catalog compilation failed, an exception would have been thrown. So if
+      # we get here, the catalog succeeded. Dump the catalog to the appropriate place
+      # and exit successfully.
+      if options[:output_file]
+        File.open(options[:output_file], 'w') { |f| f.write(to_catalog.to_json) }
+        logger.info "Wrote catalog to #{options[:output_file]}"
+      else
+        puts to_catalog.to_json
+      end
+
+      return { exitcode: EXITCODE_SUCCESS_NO_DIFFS, to: to_catalog } if options[:INTEGRATION] # For integration testing
+      EXITCODE_SUCCESS_NO_DIFFS
+    end
+
+    # --bootstrap-then-exit command
+    def self.bootstrap_then_exit(logger, catalogs_obj)
+      catalogs_obj.bootstrap_then_exit
+      return EXITCODE_SUCCESS_NO_DIFFS
+    rescue OctocatalogDiff::Errors::BootstrapError => exc
+      logger.fatal("--bootstrap-then-exit error: bootstrap failed (#{exc})")
+      return EXITCODE_FAILURE
+    end
+  end
+end

data/lib/octocatalog-diff/cli/diffs.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative '../catalog-diff/differ'
+
+module OctocatalogDiff
+  class Cli
+    # Wrapper around OctocatalogDiff::CatalogDiff::Differ to provide the logger object, set up
+    # ignores, and add additional ignores for items dependent upon the compilation directory.
+    class Diffs
+      # Constructor
+      # @param options [Hash] Options from cli/options
+      # @param logger [Logger] Logger object
+      def initialize(options, logger)
+        @options = options
+        @logger = logger
+      end
+
+      # The method to call externally, passing in the catalogs as a hash (see parameter). This
+      # sets up options and ignores and then actually performs the diffs. The result is the array
+      # of diffs.
+      # @param catalogs [Hash] { :to => OctocatalogDiff::Catalog, :from => OctocatalogDiff::Catalog }
+      # @return [Array<diffs>] Array of diffs
+      def diffs(catalogs)
+        @logger.debug 'Begin compute diffs between catalogs'
+        diff_opts = @options.merge(logger: @logger)
+
+        # Construct the actual differ object that the present one wraps
+        differ = OctocatalogDiff::CatalogDiff::Differ.new(diff_opts, catalogs[:from], catalogs[:to])
+        differ.ignore(attr: 'tags') unless @options.fetch(:include_tags, false)
+        differ.ignore(@options.fetch(:ignore, []))
+        differ.ignore_tags
+
+        # Actually perform the diff
+        diff_result = differ.diff
+        @logger.debug 'Success compute diffs between catalogs'
+        diff_result
+      end
+    end
+  end
+end